From fd00991310f2eb43f737079c5a28332865447d18 Mon Sep 17 00:00:00 2001 From: Siggi Date: Mon, 26 Oct 2026 15:11:33 +0100 Subject: [PATCH 01/33] fix(blockassembly): mark only moved-back txs not on the longest chain during a reorg A reorg folded every tx in block assembly into MarkTransactionsOnLongestChain(false), 380M records and ~8 minutes per 1-back/2-forward fork at scale with intake stalled. The mark-false set is now the losing txs plus moved-back txs not in a moveForward block that are still in block assembly. The sweep was the only repair for a tip overtaken before block assembly moved onto it: setTxMined had already cleared unmined_since on its txs. setTxMined now re-checks the chain after writing and re-marks the txs that are not mined on the new longest chain, before mined_set is written. The per-tx chain filter is extracted to model.FilterTxsNotOnLongestChain and shared by setTxMined and the invalid-block reorg check, which now reads only the invalid block's txs instead of the whole mempool. Refs #2051 --- model/filter-not-on-longest-chain.go | 116 ++++++ .../subtreeprocessor/SubtreeProcessor.go | 149 +++---- .../subtreeprocessor/reorg_mark_scope_test.go | 370 ++++++++++++++++++ services/blockvalidation/BlockValidation.go | 68 ++++ .../settxmined_overtaken_test.go | 231 +++++++++++ 5 files changed, 835 insertions(+), 99 deletions(-) create mode 100644 model/filter-not-on-longest-chain.go create mode 100644 services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go create mode 100644 services/blockvalidation/settxmined_overtaken_test.go diff --git a/model/filter-not-on-longest-chain.go b/model/filter-not-on-longest-chain.go new file mode 100644 index 0000000000..1902a0149b --- /dev/null +++ b/model/filter-not-on-longest-chain.go @@ -0,0 +1,116 @@ +package model + +import ( + "context" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/stores/utxo/fields" + "github.com/bsv-blockchain/teranode/stores/utxo/meta" + "github.com/bsv-blockchain/teranode/ulogger" + "github.com/bsv-blockchain/teranode/util" + "golang.org/x/sync/errgroup" +) + +// txBlockIDsGetter is the slice of the UTXO store FilterTxsNotOnLongestChain reads from. +type txBlockIDsGetter interface { + Get(ctx context.Context, hash *chainhash.Hash, fields ...fields.FieldName) (*meta.Data, error) +} + +// longestChainClient is the slice of the blockchain client FilterTxsNotOnLongestChain needs. +type longestChainClient interface { + GetBlockHeaders(ctx context.Context, blockHash *chainhash.Hash, numberOfHeaders uint64) ([]*BlockHeader, []*BlockHeaderMeta, error) + CheckBlockIsInCurrentChain(ctx context.Context, blockIDs []uint32) (bool, error) +} + +// FilterTxsNotOnLongestChain returns the subset of hashes that are not mined in any block on +// the current longest chain, i.e. the txs it is correct to mark not-on-longest-chain. +// +// It exists for blocks that left the longest chain after their txs were written as mined: +// a tx the block shares with the longest chain is mined there and must keep its cleared +// unmined_since, so only the rest may be re-marked. +// +// - offChainBlockID is a block known to be off the longest chain. A tx whose only block it +// is gets returned without a chain lookup. +// - anchorHash is a block on the longest chain. The 1000 headers ending at it form a fast +// path: a tx in any of those blocks is on the longest chain. Older blocks are resolved +// with CheckBlockIsInCurrentChain. +// +// Every hash must exist in the UTXO store. A missing one is an error, not a skip. +func FilterTxsNotOnLongestChain(ctx context.Context, logger ulogger.Logger, utxoStore txBlockIDsGetter, blockchainClient longestChainClient, + anchorHash *chainhash.Hash, offChainBlockID uint32, hashes []chainhash.Hash, concurrency int) ([]chainhash.Hash, error) { + notOnLongestChain := make([]chainhash.Hash, 0, len(hashes)) + + _, lastBlockHeaderMetas, err := blockchainClient.GetBlockHeaders(ctx, anchorHash, 1000) + if err != nil { + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting last block headers", err) + } + + lastBlockHeaderIDs := make(map[uint32]bool) + for _, header := range lastBlockHeaderMetas { + lastBlockHeaderIDs[header.ID] = true + } + + txMetas := make([]*meta.Data, len(hashes)) + g, gCtx := errgroup.WithContext(ctx) + // SafeSetLimit panics loudly on a zero limit rather than letting g.Go deadlock on a + // zero-capacity semaphore. + util.SafeSetLimit(logger, g, concurrency) + + for idx, hash := range hashes { + h := hash // per-iteration copy for pointer safety + g.Go(func() error { + txMeta, err := utxoStore.Get(gCtx, &h, fields.BlockIDs) + if err != nil { + return errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting transaction from utxo store", err) + } + + txMetas[idx] = txMeta + + return nil + }) + } + + if err = g.Wait(); err != nil { + return nil, err + } + + for idx, hash := range hashes { + txMeta := txMetas[idx] + if txMeta == nil { + // transaction not found, not OK + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] transaction %s not found on longest chain", hash.String()) + } + + if len(txMeta.BlockIDs) == 1 && txMeta.BlockIDs[0] == offChainBlockID { + // the transaction is only in the off-chain block, so it is definitely not on the longest chain + notOnLongestChain = append(notOnLongestChain, hash) + continue + } + + foundInRecentBlock := false + for _, blockID := range txMeta.BlockIDs { + if lastBlockHeaderIDs[blockID] { + // the transaction is still in one of the last 1000 blocks, so it is still on the longest chain + foundInRecentBlock = true + break + } + } + + if foundInRecentBlock { + continue + } + + // check BlockIDs to see if the transaction is still on the longest chain + onLongestChain, err := blockchainClient.CheckBlockIsInCurrentChain(ctx, txMeta.BlockIDs) + if err != nil { + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error checking if transaction is on longest chain", err) + } + + if !onLongestChain { + notOnLongestChain = append(notOnLongestChain, hash) + } + } + + return notOnLongestChain, nil +} diff --git a/services/blockassembly/subtreeprocessor/SubtreeProcessor.go b/services/blockassembly/subtreeprocessor/SubtreeProcessor.go index a71adf9dcf..d3b8997c90 100644 --- a/services/blockassembly/subtreeprocessor/SubtreeProcessor.go +++ b/services/blockassembly/subtreeprocessor/SubtreeProcessor.go @@ -40,7 +40,6 @@ import ( "github.com/bsv-blockchain/teranode/stores/blob/options" utxostore "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/stores/utxo/fields" - "github.com/bsv-blockchain/teranode/stores/utxo/meta" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util" "github.com/bsv-blockchain/teranode/util/retry" @@ -5083,8 +5082,6 @@ func (stp *SubtreeProcessor) reorgBlocks(ctx context.Context, moveBackBlocks []* stp.drainAndLogDiskTxMapErr("reorgBlocks_moveforward_commit") } - movedBackBlockTxMap = nil // free up memory - // Last chance to fail the whole reorg before the external, irreversible // work below (UTXO store marks, subtree announcements, SetBlockProcessedAt, // finalizeBlockProcessing): everything up to here is in-memory processor @@ -5117,23 +5114,35 @@ func (stp *SubtreeProcessor) reorgBlocks(ctx context.Context, moveBackBlocks []* // Consolidate all "mark as not on longest chain" operations: // 1. Losing conflicting transactions from moveForward blocks - // 2. All transactions currently in block assembly (chainedSubtrees + currentSubtree) + // 2. Transactions from the moved-back blocks that are not in a moveForward block and + // are still in block assembly (moveBack minus moveForward, the same net set + // BlockAssembler.Reset marks on its own path) + // + // The moved-back blocks were on the longest chain, so setTxMined cleared unmined_since + // on their txs. Those that the new chain does not mine are back in the mempool and + // must be marked here; nothing else on this path does it. // - // Assembly txs must be included because some may have entered the UTXO store via - // block validation of a non-longest-chain block (e.g., competing fork), where they - // were inserted with UnminedSince=0 (mined). These need unmined_since set. + // Pure mempool txs are deliberately NOT included (issue 2051). Every tx-creating path + // writes a mempool tx with unmined_since set, and setTxMined only clears it for a block + // on the longest chain, so a pure mempool tx needs no repair. The one way such a tx can + // look mined without being in a moveBack block is a tip that was overtaken before block + // assembly ever moved onto it; setTxMined repairs that at the source + // (BlockValidation.unmarkIfOvertaken). Folding the whole mempool in here cost a batch + // write per mempool tx on every reorg — 380M records and ~8 minutes at scale, with + // block assembly intake stalled throughout. // - // This folds the WHOLE mempool into the mark-false set on every reorg. It is only safe - // because MarkTransactionsOnLongestChain(false) keeps an existing unmined_since and - // stamps only an absent one. Before that guard (issue 1768) every reorg rewrote - // unmined_since = currentBlockHeight across the mempool, which restarted the pruner's - // parent-preservation clock for every unmined child while each parent's deleteAtHeight - // kept counting from the original spend — a one-block fork opened a + // The set is restricted to txs still in block assembly, not the whole moveBack-minus- + // moveForward set: MarkTransactionsOnLongestChain treats a missing record as fatal, and + // a tx block assembly still holds is guaranteed to exist. Reverse-cascade txs evicted by + // stp.Remove above stay unmarked, as before. + // + // MarkTransactionsOnLongestChain(false) keeps an existing unmined_since and stamps only + // an absent one. Before that guard (issue 1768) every reorg rewrote + // unmined_since = currentBlockHeight, which restarted the pruner's parent-preservation + // clock for every unmined child while each parent's deleteAtHeight kept counting from + // the original spend — a one-block fork opened a // blockHeightRetention-unminedTxRetention window in which parents were pruned from // under live children. - // - // MoveBack txs are handled by BlockAssembler.Reset (which calls - // MarkTransactionsOnLongestChain before reorgBlocks runs). subtreeNodeCount := 0 for _, subtree := range stp.chainedSubtrees { subtreeNodeCount += len(subtree.Nodes) @@ -5141,23 +5150,31 @@ func (stp *SubtreeProcessor) reorgBlocks(ctx context.Context, moveBackBlocks []* currentSubtree := stp.currentSubtree.Load() subtreeNodeCount += len(currentSubtree.Nodes) - // Collect all assembly tx hashes (coinbase placeholders are stripped inside - // partitionLongestChainMarks). - assemblyHashes := make([]chainhash.Hash, 0, subtreeNodeCount) - for _, subtree := range stp.chainedSubtrees { - for _, node := range subtree.Nodes { - assemblyHashes = append(assemblyHashes, node.Hash) + // Coinbase placeholders are never in movedBackBlockTxMap, so they drop out here. + movedBackAssemblyHashes := make([]chainhash.Hash, 0, min(len(movedBackBlockTxMap), subtreeNodeCount)) + collectMovedBack := func(nodes []subtreepkg.Node) { + for _, node := range nodes { + if _, movedBack := movedBackBlockTxMap[node.Hash]; !movedBack { + continue + } + if _, winning := winningTxSet[node.Hash]; winning { + continue + } + movedBackAssemblyHashes = append(movedBackAssemblyHashes, node.Hash) } } - for _, node := range currentSubtree.Nodes { - assemblyHashes = append(assemblyHashes, node.Hash) + for _, subtree := range stp.chainedSubtrees { + collectMovedBack(subtree.Nodes) } + collectMovedBack(currentSubtree.Nodes) + + movedBackBlockTxMap = nil // free up memory // Partition into the two mark sets. partitionLongestChainMarks guarantees the // returned slices are disjoint and de-duplicated, which is the precondition for // running the two MarkTransactionsOnLongestChain calls concurrently below without // racing on the same UTXO record. - filteredMarkOnLongestChain, allMarkFalse := partitionLongestChainMarks(markOnLongestChain, allLosingTxHashes, assemblyHashes) + filteredMarkOnLongestChain, allMarkFalse := partitionLongestChainMarks(markOnLongestChain, allLosingTxHashes, movedBackAssemblyHashes) // Run mark(true) and mark(false) concurrently — they operate on disjoint tx hash sets // (guaranteed by partitionLongestChainMarks above). @@ -5236,10 +5253,11 @@ func appendAllTxHashes(dst []chainhash.Hash, m txmap.TxMap) []chainhash.Hash { // // - markTrue: hashes to mark as on the longest chain (the moveForward "winning" txs). // - markFalse: hashes to mark as not on the longest chain (losing conflicting txs plus -// every tx still sitting in block assembly). +// the moved-back txs that are not in a moveForward block and are still in block +// assembly). // // losingTxHashes is expected to already be deduped (against the winning set and itself); -// assemblyHashes is the raw set of assembly node hashes and may contain coinbase +// assemblyHashes is a raw set of assembly node hashes and may contain coinbase // placeholders and duplicates. The returned slices are guaranteed to be: // // - de-duplicated (markFalse contains each hash at most once), and @@ -5314,79 +5332,12 @@ func (stp *SubtreeProcessor) markNotOnLongestChain(ctx context.Context, moveBack func (stp *SubtreeProcessor) checkMarkNotOnLongestChain(ctx context.Context, invalidBlock *model.Block, markNotOnLongestChain []chainhash.Hash) ([]chainhash.Hash, error) { stp.logger.Infof("[reorgBlocks] block %s we moved back is invalid, checking whether to mark transactions as not on longest chain", invalidBlock.String()) - checkMarkNotOnLongestChain := make([]chainhash.Hash, 0, len(markNotOnLongestChain)) - - _, lastBlockHeaderMetas, err := stp.blockchainClient.GetBlockHeaders(ctx, invalidBlock.Header.HashPrevBlock, 1000) + // With defaults (MaxMinedRoutines=128) the limit is always >= 1. + checkMarkNotOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, stp.logger, stp.utxoStore, stp.blockchainClient, + invalidBlock.Header.HashPrevBlock, invalidBlock.ID, markNotOnLongestChain, + max(stp.settings.UtxoStore.MaxMinedRoutines, stp.settings.UtxoStore.GetBatcherSize*2)) if err != nil { - return nil, errors.NewProcessingError("[reorgBlocks] error getting last block headers", err) - } - - lastBlockHeaderIDs := make(map[uint32]bool) - for _, header := range lastBlockHeaderMetas { - lastBlockHeaderIDs[header.ID] = true - } - - txMetas := make([]*meta.Data, len(markNotOnLongestChain)) - g, gCtx := errgroup.WithContext(ctx) - // Use SafeSetLimit (matching every other SetLimit call in this file): it - // panics loudly at setup if the computed limit is 0 (a misconfiguration with - // both MaxMinedRoutines and GetBatcherSize at 0) rather than letting g.Go - // deadlock on a zero-capacity semaphore. With defaults (MaxMinedRoutines=128) - // this is always >= 1. - util.SafeSetLimit(stp.logger, g, max(stp.settings.UtxoStore.MaxMinedRoutines, stp.settings.UtxoStore.GetBatcherSize*2)) - - // we need to check each transaction in the block we moved back and see if it is still on the longest chain or not - for idx, hash := range markNotOnLongestChain { - h := hash // per-iteration copy for pointer safety - g.Go(func() error { - txMeta, err := stp.utxoStore.Get(gCtx, &h, fields.BlockIDs) - if err != nil { - return errors.NewProcessingError("[reorgBlocks] error getting transaction from utxo store", err) - } - - txMetas[idx] = txMeta - - return nil - }) - } - - if err = g.Wait(); err != nil { - return nil, err - } - - for idx, hash := range markNotOnLongestChain { - txMeta := txMetas[idx] - if txMeta == nil { - // transaction not found, not OK - return nil, errors.NewProcessingError("[reorgBlocks] transaction %s not found on longest chain", hash.String()) - } - - if len(txMeta.BlockIDs) == 1 && txMeta.BlockIDs[0] == invalidBlock.ID { - // the transaction is only in the invalid block, so it is definitely not on the longest chain - checkMarkNotOnLongestChain = append(checkMarkNotOnLongestChain, hash) - } else { - foundInRecentBlock := false - for _, blockID := range txMeta.BlockIDs { - if lastBlockHeaderIDs[blockID] { - // the transaction is still in one of the last 1000 blocks, so it is still on the longest chain - // do not mark it as not on the longest chain - foundInRecentBlock = true - break - } - } - - if !foundInRecentBlock { - // check BlockIDs to see if the transaction is still on the longest chain - onLongestChain, err := stp.blockchainClient.CheckBlockIsInCurrentChain(ctx, txMeta.BlockIDs) - if err != nil { - return nil, errors.NewProcessingError("[reorgBlocks] error checking if transaction is on longest chain", err) - } - - if !onLongestChain { - checkMarkNotOnLongestChain = append(checkMarkNotOnLongestChain, hash) - } - } - } + return nil, errors.NewProcessingError("[reorgBlocks] error filtering transactions not on longest chain", err) } return checkMarkNotOnLongestChain, nil diff --git a/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go new file mode 100644 index 0000000000..30c21f76fa --- /dev/null +++ b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go @@ -0,0 +1,370 @@ +package subtreeprocessor + +import ( + "context" + "net/url" + "sync" + "testing" + "time" + + "github.com/bsv-blockchain/go-bt/v2" + "github.com/bsv-blockchain/go-bt/v2/bscript" + "github.com/bsv-blockchain/go-bt/v2/chainhash" + subtreepkg "github.com/bsv-blockchain/go-subtree" + "github.com/bsv-blockchain/teranode/model" + "github.com/bsv-blockchain/teranode/pkg/fileformat" + "github.com/bsv-blockchain/teranode/services/blockchain" + blob_memory "github.com/bsv-blockchain/teranode/stores/blob/memory" + utxostore "github.com/bsv-blockchain/teranode/stores/utxo" + "github.com/bsv-blockchain/teranode/stores/utxo/fields" + "github.com/bsv-blockchain/teranode/stores/utxo/meta" + "github.com/bsv-blockchain/teranode/stores/utxo/sql" + "github.com/bsv-blockchain/teranode/ulogger" + "github.com/bsv-blockchain/teranode/util/test" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// scopeMarkRecorder records the hashes of every MarkTransactionsOnLongestChain call, per +// direction, and of every Get, and then applies the call to the real store underneath. +type scopeMarkRecorder struct { + utxostore.Store + + mu sync.Mutex + markFalse map[chainhash.Hash]struct{} + markTrue map[chainhash.Hash]struct{} + gets map[chainhash.Hash]struct{} +} + +func newScopeMarkRecorder(store utxostore.Store) *scopeMarkRecorder { + return &scopeMarkRecorder{ + Store: store, + markFalse: make(map[chainhash.Hash]struct{}), + markTrue: make(map[chainhash.Hash]struct{}), + gets: make(map[chainhash.Hash]struct{}), + } +} + +func (s *scopeMarkRecorder) Get(ctx context.Context, hash *chainhash.Hash, fields ...fields.FieldName) (*meta.Data, error) { + s.mu.Lock() + s.gets[*hash] = struct{}{} + s.mu.Unlock() + + return s.Store.Get(ctx, hash, fields...) +} + +func (s *scopeMarkRecorder) gotten(hash chainhash.Hash) bool { + s.mu.Lock() + defer s.mu.Unlock() + + _, ok := s.gets[hash] + + return ok +} + +func (s *scopeMarkRecorder) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { + s.mu.Lock() + target := s.markFalse + if onLongestChain { + target = s.markTrue + } + + for _, hash := range txHashes { + target[hash] = struct{}{} + } + s.mu.Unlock() + + return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) +} + +func (s *scopeMarkRecorder) markedFalse(hash chainhash.Hash) bool { + s.mu.Lock() + defer s.mu.Unlock() + + _, ok := s.markFalse[hash] + + return ok +} + +func (s *scopeMarkRecorder) markFalseCount() int { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.markFalse) +} + +// createScopeTestTx creates a tx spending coinbaseTx output vout, at height 1. With no +// options it is an unmined mempool tx (unmined_since set). +func createScopeTestTx(t *testing.T, ctx context.Context, store utxostore.Store, vout uint32, opts ...utxostore.CreateOption) chainhash.Hash { + t.Helper() + + tx := bt.NewTx() + require.NoError(t, tx.From(coinbaseTx.TxIDChainHash().String(), vout, + coinbaseTx.Outputs[vout].LockingScript.String(), coinbaseTx.Outputs[vout].Satoshis)) + require.NoError(t, tx.AddP2PKHOutputFromAddress("1A1zP1eP5QGefi2DMPTfTL5SLmv7DivfNa", 100000000)) + tx.Inputs[0].UnlockingScript = bscript.NewFromBytes([]byte{}) + + _, _, err := store.SpendAndCreate(ctx, tx, 1, append([]utxostore.CreateOption{utxostore.WithCreateOnly()}, opts...)...) + require.NoError(t, err) + + return *tx.TxIDChainHash() +} + +// storeScopeTestSubtree stores a subtree of [coinbase, hashes...] and returns its key. +func storeScopeTestSubtree(t *testing.T, ctx context.Context, blobStore *blob_memory.Memory, hashes ...chainhash.Hash) *chainhash.Hash { + t.Helper() + + st, err := subtreepkg.NewTreeByLeafCount(4) + require.NoError(t, err) + require.NoError(t, st.AddCoinbaseNode()) + + for i, hash := range hashes { + require.NoError(t, st.AddSubtreeNode(subtreepkg.Node{Hash: hash, Fee: uint64(100 + i), SizeInBytes: 250})) //nolint:gosec + } + + stBytes, err := st.Serialize() + require.NoError(t, err) + + metaBytes, err := createSubtreeMeta(t, st).Serialize() + require.NoError(t, err) + + key := st.RootHash() + require.NoError(t, blobStore.Set(ctx, key[:], fileformat.FileTypeSubtree, stBytes)) + require.NoError(t, blobStore.Set(ctx, key[:], fileformat.FileTypeSubtreeMeta, metaBytes)) + + return key +} + +// TestReorgBlocks_MarkFalseScope pins issue 2051: a reorg marks not-on-longest-chain only +// the txs whose chain state it changes (moved back and not moved forward), never the whole +// block assembly mempool. +// +// Scenario, 1 back / 1 forward: +// +// moveBack block A: a1, shared (mined on the old chain) +// moveForward block B: shared, b1 (validated as a side chain, so b1 is still unmined) +// mempool before: m1, m2 (unmined) +func TestReorgBlocks_MarkFalseScope(t *testing.T) { + ctx := context.Background() + tSettings := test.CreateBaseTestSettings(t) + + utxoStoreURL, err := url.Parse("sqlitememory:///test") + require.NoError(t, err) + sqlStore, err := sql.New(ctx, ulogger.TestLogger{}, tSettings, utxoStoreURL) + require.NoError(t, err) + + store := newScopeMarkRecorder(sqlStore) + blobStore := blob_memory.New() + + const blockAID = uint32(10) + + a1 := createScopeTestTx(t, ctx, store, 0, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: blockAID, BlockHeight: 2, OnLongestChain: true})) + shared := createScopeTestTx(t, ctx, store, 1, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: blockAID, BlockHeight: 2, OnLongestChain: true})) + b1 := createScopeTestTx(t, ctx, store, 2) + m1 := createScopeTestTx(t, ctx, store, 3) + m2 := createScopeTestTx(t, ctx, store, 4) + + mempoolBefore := map[chainhash.Hash]uint32{} + for _, hash := range []chainhash.Hash{m1, m2} { + m, err := store.Get(ctx, &hash, fields.UnminedSince) + require.NoError(t, err) + require.NotZero(t, m.UnminedSince, "precondition: mempool tx is unmined") + mempoolBefore[hash] = m.UnminedSince + } + + _, err = sqlStore.Create(ctx, coinbaseTx2, 2) + require.NoError(t, err) + _, err = sqlStore.Create(ctx, coinbaseTx3, 2) + require.NoError(t, err) + + blockA := &model.Block{ + ID: blockAID, + Height: 2, + CoinbaseTx: coinbaseTx2, + Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, a1, shared)}, + Header: &model.BlockHeader{ + Version: 1, + HashPrevBlock: prevBlockHeader.Hash(), + HashMerkleRoot: &chainhash.Hash{}, + Timestamp: 3000000001, + Bits: model.NBit{}, + Nonce: 501, + }, + } + + blockB := &model.Block{ + ID: blockAID + 1, + Height: 2, + CoinbaseTx: coinbaseTx3, + Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, shared, b1)}, + Header: &model.BlockHeader{ + Version: 1, + HashPrevBlock: prevBlockHeader.Hash(), + HashMerkleRoot: &chainhash.Hash{}, + Timestamp: 3000000002, + Bits: model.NBit{}, + Nonce: 502, + }, + } + + mockBlockchainClient := &blockchain.Mock{} + mockBlockchainClient.On("GetBlocksMinedNotSet", mock.Anything).Return([]*model.Block{}, nil) + mockBlockchainClient.On("SetBlockProcessedAt", mock.Anything, mock.AnythingOfType("*chainhash.Hash"), mock.AnythingOfType("[]bool")).Return(nil) + mockBlockchainClient.On("GetBlockHeader", mock.Anything, mock.Anything).Return(prevBlockHeader, &model.BlockHeaderMeta{}, nil) + mockBlockchainClient.On("GetBlockIsMined", mock.Anything, mock.Anything).Return(true, nil) + + newSubtreeChan := make(chan NewSubtreeRequest, 10) + go func() { + for req := range newSubtreeChan { + if req.ErrChan != nil { + req.ErrChan <- nil + } + } + }() + t.Cleanup(func() { close(newSubtreeChan) }) + + stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, mockBlockchainClient, store, newSubtreeChan) + require.NoError(t, err) + stp.Start(ctx) + + stp.InitCurrentBlockHeader(blockA.Header) + + stp.AddBatch([]subtreepkg.Node{{Hash: m1, Fee: 300, SizeInBytes: 250}, {Hash: m2, Fee: 301, SizeInBytes: 250}}, + []*subtreepkg.TxInpoints{{}, {}}) + + require.Eventually(t, func() bool { + txMap := stp.GetCurrentTxMap() + return txMap.Exists(m1) && txMap.Exists(m2) + }, 5*time.Second, 10*time.Millisecond, "mempool txs must be in block assembly before the reorg") + + require.NoError(t, stp.Reorg([]*model.Block{blockA}, []*model.Block{blockB})) + require.Equal(t, blockB.Header.Hash(), stp.GetCurrentBlockHeader().Hash()) + + finalTxMap := stp.GetCurrentTxMap() + require.True(t, finalTxMap.Exists(a1), "a1 is back in the mempool after its block was moved back") + require.True(t, finalTxMap.Exists(m1)) + require.True(t, finalTxMap.Exists(m2)) + + // The mark-false set is exactly what the reorg moved off the chain. + require.True(t, store.markedFalse(a1), "a moved-back tx not in the new chain must be marked not on the longest chain") + require.False(t, store.markedFalse(shared), "a tx mined by the moveForward block must not be marked false") + require.False(t, store.markedFalse(m1), "a pure mempool tx must not be re-marked on a reorg") + require.False(t, store.markedFalse(m2), "a pure mempool tx must not be re-marked on a reorg") + require.Equal(t, 1, store.markFalseCount(), "only a1 changed chain state in the not-on-longest-chain direction") + + // End state in the store. + a1Meta, err := store.Get(ctx, &a1, fields.UnminedSince) + require.NoError(t, err) + require.NotZero(t, a1Meta.UnminedSince, "a1 is unmined again") + + b1Meta, err := store.Get(ctx, &b1, fields.UnminedSince) + require.NoError(t, err) + require.Zero(t, b1Meta.UnminedSince, "b1 is mined on the new longest chain") + + sharedMeta, err := store.Get(ctx, &shared, fields.UnminedSince) + require.NoError(t, err) + require.Zero(t, sharedMeta.UnminedSince, "shared stays mined") + + for hash, before := range mempoolBefore { + m, err := store.Get(ctx, &hash, fields.UnminedSince) + require.NoError(t, err) + require.Equal(t, before, m.UnminedSince, "a pure mempool tx keeps its unmined_since") + } +} + +// TestReorgBlocks_InvalidMoveBackChecksOnlyItsOwnTxs covers the 1 back / 0 forward reorg of +// an invalid block (block 992 on 2026-10-26). The per-tx chain check must run over the +// invalid block's txs only, not every tx in block assembly: on that day it was ~880M +// per-tx store reads, and no block was produced while it ran. +// +// moveBack block A (invalid): a1 (only in A), dup (also in valid block 5) +// mempool before: m1 (unmined) +func TestReorgBlocks_InvalidMoveBackChecksOnlyItsOwnTxs(t *testing.T) { + ctx := context.Background() + tSettings := test.CreateBaseTestSettings(t) + if tSettings.UtxoStore.MaxMinedRoutines < 4 { + tSettings.UtxoStore.MaxMinedRoutines = 4 + } + + utxoStoreURL, err := url.Parse("sqlitememory:///test") + require.NoError(t, err) + sqlStore, err := sql.New(ctx, ulogger.TestLogger{}, tSettings, utxoStoreURL) + require.NoError(t, err) + + store := newScopeMarkRecorder(sqlStore) + blobStore := blob_memory.New() + + const ( + invalidBlockID = uint32(20) + validBlockID = uint32(5) + ) + + a1 := createScopeTestTx(t, ctx, store, 0, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: invalidBlockID, BlockHeight: 2, OnLongestChain: true})) + dup := createScopeTestTx(t, ctx, store, 1, utxostore.WithMinedBlockInfo( + utxostore.MinedBlockInfo{BlockID: validBlockID, BlockHeight: 1, OnLongestChain: true}, + utxostore.MinedBlockInfo{BlockID: invalidBlockID, BlockHeight: 2, OnLongestChain: true}, + )) + m1 := createScopeTestTx(t, ctx, store, 2) + + _, err = sqlStore.Create(ctx, coinbaseTx2, 2) + require.NoError(t, err) + + blockA := &model.Block{ + ID: invalidBlockID, + Height: 2, + CoinbaseTx: coinbaseTx2, + Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, a1, dup)}, + Header: &model.BlockHeader{ + Version: 1, + HashPrevBlock: prevBlockHeader.Hash(), + HashMerkleRoot: &chainhash.Hash{}, + Timestamp: 3000000011, + Bits: model.NBit{}, + Nonce: 511, + }, + } + + mockBlockchainClient := &blockchain.Mock{} + mockBlockchainClient.On("GetBlocksMinedNotSet", mock.Anything).Return([]*model.Block{}, nil) + mockBlockchainClient.On("SetBlockProcessedAt", mock.Anything, mock.AnythingOfType("*chainhash.Hash"), mock.AnythingOfType("[]bool")).Return(nil) + mockBlockchainClient.On("GetBlockHeader", mock.Anything, mock.Anything).Return(prevBlockHeader, &model.BlockHeaderMeta{Invalid: true}, nil) + mockBlockchainClient.On("GetBlockHeaders", mock.Anything, mock.Anything, mock.Anything). + Return([]*model.BlockHeader{}, []*model.BlockHeaderMeta{{ID: validBlockID}}, nil) + // Only reached for a tx in neither the invalid block alone nor a recent block, i.e. a + // mempool tx that should never have been checked. + mockBlockchainClient.On("CheckBlockIsInCurrentChain", mock.Anything, mock.Anything).Return(false, nil) + mockBlockchainClient.On("GetBlock", mock.Anything, mock.Anything).Return(&model.Block{Header: prevBlockHeader, Height: 1}, nil) + mockBlockchainClient.On("GetBlockIsMined", mock.Anything, mock.Anything).Return(true, nil) + + newSubtreeChan := make(chan NewSubtreeRequest, 10) + go func() { + for req := range newSubtreeChan { + if req.ErrChan != nil { + req.ErrChan <- nil + } + } + }() + t.Cleanup(func() { close(newSubtreeChan) }) + + stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, mockBlockchainClient, store, newSubtreeChan) + require.NoError(t, err) + stp.Start(ctx) + + stp.InitCurrentBlockHeader(blockA.Header) + + stp.AddBatch([]subtreepkg.Node{{Hash: m1, Fee: 300, SizeInBytes: 250}}, []*subtreepkg.TxInpoints{{}}) + + require.Eventually(t, func() bool { + return stp.GetCurrentTxMap().Exists(m1) + }, 5*time.Second, 10*time.Millisecond, "mempool tx must be in block assembly before the reorg") + + require.NoError(t, stp.Reorg([]*model.Block{blockA}, []*model.Block{})) + + require.True(t, store.gotten(a1), "the invalid block's txs are checked") + require.True(t, store.gotten(dup), "the invalid block's txs are checked") + require.False(t, store.gotten(m1), "a pure mempool tx must not be read by the invalid-block check") + + require.True(t, store.markedFalse(a1), "a tx only in the invalid block is not on the longest chain") + require.False(t, store.markedFalse(dup), "a tx also in a valid block stays on the longest chain") + require.False(t, store.markedFalse(m1), "a pure mempool tx must not be re-marked") +} diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 504b6078be..5ce2074c52 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1606,6 +1606,14 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return errors.NewProcessingError("[setTxMined][%s] error updating tx mined status", block.Hash().String(), err) } + // Must run before SetBlockMinedSet below: block assembly waits on mined_set before it + // acts on this block, so the repair is in place before anything can build on it. + if onLongestChain && (len(unsetMined) == 0 || !unsetMined[0]) { + if err = u.unmarkIfOvertaken(ctx, block, blockHeaderMeta.ID); err != nil { + return err + } + } + // Close mmap-backed subtrees and clear to free memory. Routed through // ReleaseSubtreeNodes so the entries are nil-ed under the block's subtree // mutex: this block may still be reachable via lastValidatedBlocks, whose @@ -1626,6 +1634,66 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return nil } +// unmarkIfOvertaken repairs the txs of a block that left the longest chain while its +// setTxMined was running. +// +// setTxMined decides onLongestChain once, before writing. If the block was the tip then, +// the write cleared unmined_since on its txs. If a competing chain overtook it before block +// assembly moved onto it, block assembly reaches the new tip by a pure catch-up, never sees +// this block as a moveBack block, and nothing else restores unmined_since. Those txs would +// then sit in the mempool looking mined: prunable, and invisible to loadUnminedTransactions. +// +// Only the txs not mined on the new longest chain are re-marked; a tx shared with the +// winning chain is mined there and keeps its cleared unmined_since. This is the source-side +// repair that lets block assembly's reorg mark only the txs a reorg actually moves +// (issue 2051). +func (u *BlockValidation) unmarkIfOvertaken(ctx context.Context, block *model.Block, blockID uint32) error { + stillOnLongestChain, err := u.blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{blockID}) + if err != nil { + return errors.NewServiceError("[setTxMined][%s] failed to re-check if block is on longest chain", block.Hash().String(), err) + } + + if stillOnLongestChain { + return nil + } + + bestBlockHeader, _, err := u.blockchainClient.GetBestBlockHeader(ctx) + if err != nil { + return errors.NewServiceError("[setTxMined][%s] failed to get best block header", block.Hash().String(), err) + } + + txHashes := make([]chainhash.Hash, 0, block.TransactionCount) + + for _, subtree := range block.SubtreeSlices { + if subtree == nil { + return errors.NewProcessingError("[setTxMined][%s] missing subtree while re-marking an overtaken block", block.Hash().String()) + } + + for _, node := range subtree.Nodes { + if !node.Hash.Equal(subtreepkg.CoinbasePlaceholderHashValue) { + txHashes = append(txHashes, node.Hash) + } + } + } + + notOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, u.logger, u.utxoStore, u.blockchainClient, + bestBlockHeader.Hash(), blockID, txHashes, + max(u.settings.UtxoStore.MaxMinedRoutines, u.settings.UtxoStore.GetBatcherSize*2)) + if err != nil { + return errors.NewProcessingError("[setTxMined][%s] error filtering txs of overtaken block", block.Hash().String(), err) + } + + u.logger.Warnf("[setTxMined][%s] block left the longest chain during setTxMined, marking %d of %d txs as not on longest chain", block.Hash().String(), len(notOnLongestChain), len(txHashes)) + + if len(notOnLongestChain) > 0 { + if err = u.utxoStore.MarkTransactionsOnLongestChain(ctx, notOnLongestChain, false); err != nil { + return errors.NewProcessingError("[setTxMined][%s] error marking txs of overtaken block as not on longest chain", block.Hash().String(), err) + } + } + + return nil +} + // isParentMined verifies if a block's parent has been successfully mined and committed to the blockchain. // // This function performs a critical validation step in the block processing pipeline by diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go new file mode 100644 index 0000000000..c143d0b35c --- /dev/null +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -0,0 +1,231 @@ +package blockvalidation + +import ( + "context" + "net/url" + "sync" + "testing" + "time" + + "github.com/bsv-blockchain/go-bt/v2" + "github.com/bsv-blockchain/go-bt/v2/bscript" + "github.com/bsv-blockchain/go-bt/v2/chainhash" + subtreepkg "github.com/bsv-blockchain/go-subtree" + "github.com/bsv-blockchain/teranode/model" + "github.com/bsv-blockchain/teranode/pkg/fileformat" + "github.com/bsv-blockchain/teranode/services/blockchain" + blockchain_store "github.com/bsv-blockchain/teranode/stores/blockchain" + utxostore "github.com/bsv-blockchain/teranode/stores/utxo" + "github.com/bsv-blockchain/teranode/stores/utxo/fields" + "github.com/bsv-blockchain/teranode/ulogger" + "github.com/bsv-blockchain/teranode/util/test" + "github.com/stretchr/testify/require" +) + +// overtakeOnSetMined runs fn once, right after the first SetMinedMulti call for blockID +// has been applied. It lets a test move the chain tip while setTxMined is in flight, +// which is the window between setTxMined's onLongestChain check and its completion. +type overtakeOnSetMined struct { + utxostore.Store + blockID uint32 + once sync.Once + fn func() +} + +func (s *overtakeOnSetMined) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { + res, err := s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) + if err == nil && minedBlockInfo.BlockID == s.blockID { + s.once.Do(s.fn) + } + + return res, err +} + +// coinbaseOnlyBlock builds a mined block with no subtrees on parent. tag makes the +// coinbase, and so the block, unique. +func coinbaseOnlyBlock(t *testing.T, parent *chainhash.Hash, nBits *model.NBit, height uint32, tag byte) *model.Block { + t.Helper() + + coinbaseTx := bt.NewTx() + require.NoError(t, coinbaseTx.From("0000000000000000000000000000000000000000000000000000000000000000", 0xffffffff, "", 0)) + coinbaseTx.Inputs[0].UnlockingScript = bscript.NewFromBytes([]byte{0x03, byte(height), 0x00, 0x00, tag, '/', 'T', 'e', 's', 't'}) + require.NoError(t, coinbaseTx.AddP2PKHOutputFromAddress("1MUMxUTXcPQ1kAqB7MtJWneeAwVW4cHzzp", 50*100000000)) + + header := mineBlockHeader(t, parent, coinbaseTx.TxIDChainHash(), nBits) + + block, err := model.NewBlock(header, coinbaseTx, []*chainhash.Hash{}, 1, uint64(coinbaseTx.Size()), height, 0) //nolint:gosec + require.NoError(t, err) + + return block +} + +// TestSetTxMinedStatus_BlockOvertakenDuringSetMined pins the repair for a tip that is +// overtaken while its setTxMined runs (issue 2051). +// +// setTxMined decides onLongestChain once, before it writes. If block B is the tip at that +// point, the write clears unmined_since on B's txs. If a competing chain then overtakes B +// before block assembly ever moves onto B, block assembly reaches the new tip by a pure +// catch-up and never sees B as a moveBack block, so nothing restores unmined_since on B's +// txs. They stay in block assembly's mempool looking mined, which makes them prunable and +// invisible to loadUnminedTransactions. setTxMined must notice B fell off the chain and +// re-mark the txs that are not mined on the new chain. A tx that B shares with the winning +// chain is mined there and must stay cleared. +func TestSetTxMinedStatus_BlockOvertakenDuringSetMined(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + innerUtxoStore, subtreeValidationClient, _, txStore, subtreeStore, deferFunc := setup(t) + defer deferFunc() + + tSettings := test.CreateBaseTestSettings(t) + blockChainStore, err := blockchain_store.NewStore(ulogger.TestLogger{}, &url.URL{Scheme: "sqlitememory"}, tSettings) + require.NoError(t, err) + blockchainClient, err := blockchain.NewLocalClient(ulogger.TestLogger{}, tSettings, blockChainStore, nil, nil) + require.NoError(t, err) + + // Two unrelated mempool txs, created unmined. The sql store stamps unmined_since with + // the create height, so create above 0 to make "unmined" observable. + coinbaseB, bOnlyTx, _, _ := createCoinbaseAndChildTx(t) + _, sharedTx, _, _ := createCoinbaseAndChildTx(t) + + _, _, err = innerUtxoStore.SpendAndCreate(ctx, bOnlyTx, 1, utxostore.WithCreateOnly()) + require.NoError(t, err) + _, _, err = innerUtxoStore.SpendAndCreate(ctx, sharedTx, 1, utxostore.WithCreateOnly()) + require.NoError(t, err) + + for _, h := range []*chainhash.Hash{bOnlyTx.TxIDChainHash(), sharedTx.TxIDChainHash()} { + m, err := innerUtxoStore.Get(ctx, h, fields.UnminedSince) + require.NoError(t, err) + require.NotZero(t, m.UnminedSince, "precondition: mempool tx is created unmined") + } + + // Block B on genesis: coinbase, bOnly, shared. + subtree, err := subtreepkg.NewTreeByLeafCount(4) + require.NoError(t, err) + require.NoError(t, subtree.AddCoinbaseNode()) + require.NoError(t, subtree.AddNode(*bOnlyTx.TxIDChainHash(), uint64(bOnlyTx.Size()), 0)) //nolint:gosec + require.NoError(t, subtree.AddNode(*sharedTx.TxIDChainHash(), uint64(sharedTx.Size()), 0)) //nolint:gosec + + subtreeBytes, err := subtree.Serialize() + require.NoError(t, err) + require.NoError(t, subtreeStore.Set(ctx, subtree.RootHash()[:], fileformat.FileTypeSubtree, subtreeBytes)) + + replicated := subtree.Duplicate() + replicated.ReplaceRootNode(coinbaseB.TxIDChainHash(), 0, uint64(coinbaseB.Size())) //nolint:gosec + + nBits, err := model.NewNBitFromString("207fffff") + require.NoError(t, err) + + headerB := mineBlockHeader(t, tSettings.ChainCfgParams.GenesisHash, replicated.RootHash(), nBits) + totalSize := uint64(coinbaseB.Size() + bOnlyTx.Size() + sharedTx.Size()) //nolint:gosec + blockB, err := model.NewBlock(headerB, coinbaseB, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), totalSize, 1, 0) //nolint:gosec + require.NoError(t, err) + + require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test")) + + _, metaB, err := blockchainClient.GetBlockHeader(ctx, blockB.Hash()) + require.NoError(t, err) + + onChain, err := blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{metaB.ID}) + require.NoError(t, err) + require.True(t, onChain, "precondition: B is the tip when setTxMined starts") + + // The competing chain B' -> C' on genesis, which overtakes B mid-setTxMined. B' also + // mines sharedTx, so sharedTx is mined on the winning chain. + blockB2 := coinbaseOnlyBlock(t, tSettings.ChainCfgParams.GenesisHash, nBits, 1, 0xb2) + blockC2 := coinbaseOnlyBlock(t, blockB2.Hash(), nBits, 2, 0xc2) + + utxoStore := &overtakeOnSetMined{ + Store: innerUtxoStore, + blockID: metaB.ID, + fn: func() { + // Header timestamps must not decrease along the chain. + time.Sleep(10 * time.Millisecond) + + require.NoError(t, blockchainClient.AddBlock(ctx, blockB2, "test")) + require.NoError(t, blockchainClient.AddBlock(ctx, blockC2, "test")) + + _, metaB2, err := blockchainClient.GetBlockHeader(ctx, blockB2.Hash()) + require.NoError(t, err) + + _, err = innerUtxoStore.SetMinedMulti(ctx, []*chainhash.Hash{sharedTx.TxIDChainHash()}, utxostore.MinedBlockInfo{ + BlockID: metaB2.ID, + BlockHeight: 1, + SubtreeIdx: 0, + OnLongestChain: true, + }) + require.NoError(t, err) + }, + } + + blockValidation := NewBlockValidation(ctx, ulogger.TestLogger{}, tSettings, blockchainClient, subtreeStore, txStore, utxoStore, nil, subtreeValidationClient) + + require.NoError(t, blockValidation.setTxMinedStatus(ctx, blockB.Hash())) + + onChain, err = blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{metaB.ID}) + require.NoError(t, err) + require.False(t, onChain, "precondition: the competing chain overtook B during setTxMined") + + bOnlyMeta, err := innerUtxoStore.Get(ctx, bOnlyTx.TxIDChainHash(), fields.UnminedSince, fields.BlockIDs) + require.NoError(t, err) + require.Equal(t, []uint32{metaB.ID}, bOnlyMeta.BlockIDs) + require.NotZero(t, bOnlyMeta.UnminedSince, "a tx mined only in the overtaken block must be unmined again") + + sharedMeta, err := innerUtxoStore.Get(ctx, sharedTx.TxIDChainHash(), fields.UnminedSince, fields.BlockIDs) + require.NoError(t, err) + require.Zero(t, sharedMeta.UnminedSince, "a tx also mined on the winning chain must stay mined") +} + +// TestSetTxMinedStatus_BlockStillOnChain_NoRemark is the control: when B is still the tip +// after setTxMined writes, its txs stay mined. +func TestSetTxMinedStatus_BlockStillOnChain_NoRemark(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + utxoStore, subtreeValidationClient, _, txStore, subtreeStore, deferFunc := setup(t) + defer deferFunc() + + tSettings := test.CreateBaseTestSettings(t) + blockChainStore, err := blockchain_store.NewStore(ulogger.TestLogger{}, &url.URL{Scheme: "sqlitememory"}, tSettings) + require.NoError(t, err) + blockchainClient, err := blockchain.NewLocalClient(ulogger.TestLogger{}, tSettings, blockChainStore, nil, nil) + require.NoError(t, err) + + coinbaseB, bOnlyTx, _, _ := createCoinbaseAndChildTx(t) + + _, _, err = utxoStore.SpendAndCreate(ctx, bOnlyTx, 1, utxostore.WithCreateOnly()) + require.NoError(t, err) + + subtree, err := subtreepkg.NewTreeByLeafCount(2) + require.NoError(t, err) + require.NoError(t, subtree.AddCoinbaseNode()) + require.NoError(t, subtree.AddNode(*bOnlyTx.TxIDChainHash(), uint64(bOnlyTx.Size()), 0)) //nolint:gosec + + subtreeBytes, err := subtree.Serialize() + require.NoError(t, err) + require.NoError(t, subtreeStore.Set(ctx, subtree.RootHash()[:], fileformat.FileTypeSubtree, subtreeBytes)) + + replicated := subtree.Duplicate() + replicated.ReplaceRootNode(coinbaseB.TxIDChainHash(), 0, uint64(coinbaseB.Size())) //nolint:gosec + + nBits, err := model.NewNBitFromString("207fffff") + require.NoError(t, err) + + headerB := mineBlockHeader(t, tSettings.ChainCfgParams.GenesisHash, replicated.RootHash(), nBits) + blockB, err := model.NewBlock(headerB, coinbaseB, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), uint64(coinbaseB.Size()+bOnlyTx.Size()), 1, 0) //nolint:gosec + require.NoError(t, err) + + require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test")) + + blockValidation := NewBlockValidation(ctx, ulogger.TestLogger{}, tSettings, blockchainClient, subtreeStore, txStore, utxoStore, nil, subtreeValidationClient) + + require.NoError(t, blockValidation.setTxMinedStatus(ctx, blockB.Hash())) + + bOnlyMeta, err := utxoStore.Get(ctx, bOnlyTx.TxIDChainHash(), fields.UnminedSince) + require.NoError(t, err) + require.Zero(t, bOnlyMeta.UnminedSince, "a tx mined in the tip must stay mined") +} From 26a1e3fdac706d3e5a22ed1a06b4ab8d023103a0 Mon Sep 17 00:00:00 2001 From: Siggi Date: Mon, 26 Oct 2026 19:55:17 +0100 Subject: [PATCH 02/33] fix(blockvalidation): repair overtaken tips on the setMined worker A tip that is overtaken before block assembly moves onto it keeps the unmined_since its setTxMined cleared: block assembly reaches the new tip by a pure catch-up and never sees it as a moveBack block. Block assembly's reorg used to cover this by re-marking its whole mempool; with that sweep narrowed, block validation repairs it. Block validation tracks every block setTxMined writes as on the longest chain, recording it before the write. When one has left the chain, its txs that still look mined and are not mined on the new chain are re-marked not on the longest chain. A re-marked block that comes back onto the chain has its txs marked mined again; a block that comes back with nothing re-marked is simply eligible for repair again. Invalid blocks and blocks 1000 or more below the tip are dropped. The repair runs on the setMined worker, between blocks, as bounded steps: one subtree load or one slice of txs (blockvalidation_tipRepairSliceSize, default 250000) per step, holding only the current subtree's tx hashes. No setTxMined write of this process lands inside a step, so a tx the overtaken block shares with another block ends correct in either order. A new setTxMined write of a block abandons any repair in progress for it. Steps are requested by best-chain notifications and completed writes, and by the periodic tick while the last step failed. Transient failures are retried for as long as it takes; a block whose block or subtrees are gone is dropped after ten attempts. After a restart the tracker is seeded, once the process marks a block mined, from the main chain and from mined fork blocks near the tip. The main- chain id set is extended rather than refetched when the best block only extends the previous one. A subtree that is gone or unreadable is skipped with an error log so the rest of the block is still repaired, and marking a block mined again only rewrites the txs that were actually re-marked. Limitations: the repair is serialised with this process's setMined worker only, so it assumes one block validation instance writes setTxMined for the chain. Repair state is in memory; a re-marked block that comes back onto the chain after a restart keeps its txs flagged unmined until loadUnminedTransactions' fix pass, and fork blocks deeper than ten below the tip at restart are not re-seeded. The repair races the pruner, so its writes use a new MarkTransactionsOnLongestChain option, WithSkipMissing, that makes a record pruned since it was read a logged skip instead of fatal. Every other caller keeps the fatal check. The option is variadic: callers are unchanged, and wrappers pass it through. SubtreeProcessor.reset keeps its whole-assembly sweep as the catch-all for a reset that lands before a repair. FilterTxsNotOnLongestChain reads BlockIDs and UnminedSince in batches through BatchDecorate, skips txs that are already unmined, takes the caller's main-chain id set, and resolves each distinct set of block ids with one CheckBlockIsInCurrentChain call, so a repair writes only what changes and a fork storm costs a handful of chain lookups, not one per tx. The test MockStore implements GetLastNBlocks for the seed. Refs #2051 --- .../services/blockvalidation_settings.md | 1 + model/filter-not-on-longest-chain.go | 193 +++-- services/blockassembly/liveness_reset_test.go | 4 +- .../reset_pruned_moveback_test.go | 4 +- .../subtreeprocessor/SubtreeProcessor.go | 28 +- .../reorg_collect_all_txs_test.go | 2 +- .../reorg_mark_partition_test.go | 7 +- .../subtreeprocessor/reorg_mark_scope_test.go | 15 +- services/blockpersister/Server_test.go | 2 +- services/blockvalidation/BlockValidation.go | 138 ++-- .../blockvalidation/overtaken_tip_repair.go | 641 ++++++++++++++++ .../settxmined_overtaken_test.go | 720 +++++++++++++++--- settings.conf | 5 + settings/blockvalidation_settings.go | 1 + settings/settings.go | 1 + stores/blockchain/mock.go | 42 +- stores/txmetacache/txmetacache.go | 4 +- stores/utxo/Interface.go | 34 +- stores/utxo/aerospike/longest_chain.go | 11 +- stores/utxo/factory/utxo_test.go | 2 +- stores/utxo/logger/logger.go | 4 +- stores/utxo/logger/logger_test.go | 2 +- stores/utxo/mock.go | 2 +- stores/utxo/nullstore/nullstore.go | 2 +- .../sql/longest_chain_skip_missing_test.go | 89 +++ stores/utxo/sql/sql.go | 11 +- 26 files changed, 1707 insertions(+), 258 deletions(-) create mode 100644 services/blockvalidation/overtaken_tip_repair.go create mode 100644 stores/utxo/sql/longest_chain_skip_missing_test.go diff --git a/docs/references/settings/services/blockvalidation_settings.md b/docs/references/settings/services/blockvalidation_settings.md index f6418ff5f3..6d8cc21e40 100644 --- a/docs/references/settings/services/blockvalidation_settings.md +++ b/docs/references/settings/services/blockvalidation_settings.md @@ -69,6 +69,7 @@ loader (0, false and false), so for these keys the two agree. The | MaxBlocksBehindBlockAssembly | int | 20 | blockvalidation_maxBlocksBehindBlockAssembly | **CRITICAL** - Max blocks behind block assembly | | PeriodicProcessingInterval | time.Duration | 1m | blockvalidation_periodic_processing_interval | Periodic processing interval | | RecentBlockIDsLimit | uint64 | 50000 | blockvalidation_recentBlockIDsLimit | **CRITICAL** - Fast-path double-spend checking window | +| TipRepairSliceSize | int | 250000 | blockvalidation_tipRepairSliceSize | Transactions per overtaken-tip repair step on the setMined worker; bounds how long a step delays the next block's mined status | | MaxParallelForks | int | 4 | blockvalidation_max_parallel_forks | Maximum parallel fork processing | | MaxTrackedForks | int | 1000 | blockvalidation_max_tracked_forks | Maximum total forks tracked | | NearForkThreshold | int | 0 | blockvalidation_near_fork_threshold | Near fork detection (0=coinbase maturity/2) | diff --git a/model/filter-not-on-longest-chain.go b/model/filter-not-on-longest-chain.go index 1902a0149b..34e8063f95 100644 --- a/model/filter-not-on-longest-chain.go +++ b/model/filter-not-on-longest-chain.go @@ -2,95 +2,114 @@ package model import ( "context" + "fmt" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/settings" + "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/stores/utxo/fields" - "github.com/bsv-blockchain/teranode/stores/utxo/meta" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util" "golang.org/x/sync/errgroup" ) -// txBlockIDsGetter is the slice of the UTXO store FilterTxsNotOnLongestChain reads from. -type txBlockIDsGetter interface { - Get(ctx context.Context, hash *chainhash.Hash, fields ...fields.FieldName) (*meta.Data, error) +// filterNotOnLongestChainBatchSize is how many records the filters here read per +// BatchDecorate call. +const filterNotOnLongestChainBatchSize = 1024 + +// txMetaBatchReader is the slice of the UTXO store FilterTxsNotOnLongestChain reads from. +type txMetaBatchReader interface { + BatchDecorate(ctx context.Context, unresolvedMetaDataSlice []*utxo.UnresolvedMetaData, fields ...fields.FieldName) error } // longestChainClient is the slice of the blockchain client FilterTxsNotOnLongestChain needs. type longestChainClient interface { - GetBlockHeaders(ctx context.Context, blockHash *chainhash.Hash, numberOfHeaders uint64) ([]*BlockHeader, []*BlockHeaderMeta, error) CheckBlockIsInCurrentChain(ctx context.Context, blockIDs []uint32) (bool, error) } -// FilterTxsNotOnLongestChain returns the subset of hashes that are not mined in any block on -// the current longest chain, i.e. the txs it is correct to mark not-on-longest-chain. -// -// It exists for blocks that left the longest chain after their txs were written as mined: -// a tx the block shares with the longest chain is mined there and must keep its cleared -// unmined_since, so only the rest may be re-marked. -// -// - offChainBlockID is a block known to be off the longest chain. A tx whose only block it -// is gets returned without a chain lookup. -// - anchorHash is a block on the longest chain. The 1000 headers ending at it form a fast -// path: a tx in any of those blocks is on the longest chain. Older blocks are resolved -// with CheckBlockIsInCurrentChain. -// -// Every hash must exist in the UTXO store. A missing one is an error, not a skip. -func FilterTxsNotOnLongestChain(ctx context.Context, logger ulogger.Logger, utxoStore txBlockIDsGetter, blockchainClient longestChainClient, - anchorHash *chainhash.Hash, offChainBlockID uint32, hashes []chainhash.Hash, concurrency int) ([]chainhash.Hash, error) { - notOnLongestChain := make([]chainhash.Hash, 0, len(hashes)) +// recentChainHeadersClient is the slice of the blockchain client RecentChainBlockIDs needs. +type recentChainHeadersClient interface { + GetBlockHeaders(ctx context.Context, blockHash *chainhash.Hash, numberOfHeaders uint64) ([]*BlockHeader, []*BlockHeaderMeta, error) +} - _, lastBlockHeaderMetas, err := blockchainClient.GetBlockHeaders(ctx, anchorHash, 1000) +// RecentChainBlockIDs returns the ids of the n blocks ending at blockHash, for +// FilterTxsNotOnLongestChain's fast path when blockHash is on the longest chain. +func RecentChainBlockIDs(ctx context.Context, blockchainClient recentChainHeadersClient, blockHash *chainhash.Hash, n uint64) (map[uint32]bool, error) { + _, metas, err := blockchainClient.GetBlockHeaders(ctx, blockHash, n) if err != nil { - return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting last block headers", err) + return nil, errors.NewProcessingError("[RecentChainBlockIDs] error getting last block headers", err) } - lastBlockHeaderIDs := make(map[uint32]bool) - for _, header := range lastBlockHeaderMetas { - lastBlockHeaderIDs[header.ID] = true + ids := make(map[uint32]bool, len(metas)) + for _, meta := range metas { + ids[meta.ID] = true } - txMetas := make([]*meta.Data, len(hashes)) - g, gCtx := errgroup.WithContext(ctx) - // SafeSetLimit panics loudly on a zero limit rather than letting g.Go deadlock on a - // zero-capacity semaphore. - util.SafeSetLimit(logger, g, concurrency) - - for idx, hash := range hashes { - h := hash // per-iteration copy for pointer safety - g.Go(func() error { - txMeta, err := utxoStore.Get(gCtx, &h, fields.BlockIDs) - if err != nil { - return errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting transaction from utxo store", err) - } + return ids, nil +} - txMetas[idx] = txMeta +// FilterTxsNotOnLongestChain returns the subset of hashes that still look mined +// (unmined_since absent) but are not mined in any block on the current longest chain: +// exactly the txs whose MarkTransactionsOnLongestChain(false) would change something. +// +// It exists for blocks that left the longest chain after their txs were written as mined. +// A tx the block shares with the longest chain is mined there and must keep its cleared +// unmined_since; a tx that already has unmined_since needs no write at all. +// +// - offChainBlockID is a block the caller has established is off the longest chain. A tx +// whose only block it is gets returned without a chain lookup. The caller owns that +// claim: if the chain can move back onto the block, re-check before marking. +// - recentMainChainIDs is a set of block ids on the longest chain, typically the most +// recent ones (RecentChainBlockIDs). A tx in any of them is on the longest chain without +// a lookup. Other block id sets are resolved with CheckBlockIsInCurrentChain, once per +// distinct set: the txs of one block share a handful of them. +// +// A hash missing from the UTXO store is an error, unless skipMissing is set: a caller +// re-marking an old block's txs passes it, since a tx pruned since then has nothing to mark. +func FilterTxsNotOnLongestChain(ctx context.Context, logger ulogger.Logger, tSettings *settings.Settings, utxoStore txMetaBatchReader, + blockchainClient longestChainClient, recentMainChainIDs map[uint32]bool, offChainBlockID uint32, hashes []chainhash.Hash, skipMissing bool) ([]chainhash.Hash, error) { + notOnLongestChain := make([]chainhash.Hash, 0, len(hashes)) + inCurrentChain := make(map[string]bool) // CheckBlockIsInCurrentChain answers, by block id set - return nil - }) + items, err := batchReadTxMeta(ctx, logger, tSettings, utxoStore, hashes, fields.BlockIDs, fields.UnminedSince) + if err != nil { + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting transactions from utxo store", err) } - if err = g.Wait(); err != nil { - return nil, err - } + for _, item := range items { + if item.Err != nil { + if skipMissing && errors.Is(item.Err, errors.ErrTxNotFound) { + continue + } - for idx, hash := range hashes { - txMeta := txMetas[idx] + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error getting transaction %s from utxo store", item.Hash.String(), item.Err) + } + + txMeta := item.Data if txMeta == nil { + if skipMissing { + continue + } + // transaction not found, not OK - return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] transaction %s not found on longest chain", hash.String()) + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] transaction %s not found on longest chain", item.Hash.String()) + } + + if txMeta.UnminedSince != 0 { + // already unmined: marking it would change nothing (issue 1768 keeps an existing value) + continue } if len(txMeta.BlockIDs) == 1 && txMeta.BlockIDs[0] == offChainBlockID { // the transaction is only in the off-chain block, so it is definitely not on the longest chain - notOnLongestChain = append(notOnLongestChain, hash) + notOnLongestChain = append(notOnLongestChain, item.Hash) continue } foundInRecentBlock := false for _, blockID := range txMeta.BlockIDs { - if lastBlockHeaderIDs[blockID] { + if recentMainChainIDs[blockID] { // the transaction is still in one of the last 1000 blocks, so it is still on the longest chain foundInRecentBlock = true break @@ -102,15 +121,79 @@ func FilterTxsNotOnLongestChain(ctx context.Context, logger ulogger.Logger, utxo } // check BlockIDs to see if the transaction is still on the longest chain - onLongestChain, err := blockchainClient.CheckBlockIsInCurrentChain(ctx, txMeta.BlockIDs) - if err != nil { - return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error checking if transaction is on longest chain", err) + key := fmt.Sprint(txMeta.BlockIDs) + + onLongestChain, known := inCurrentChain[key] + if !known { + var err error + if onLongestChain, err = blockchainClient.CheckBlockIsInCurrentChain(ctx, txMeta.BlockIDs); err != nil { + return nil, errors.NewProcessingError("[FilterTxsNotOnLongestChain] error checking if transaction is on longest chain", err) + } + + inCurrentChain[key] = onLongestChain } if !onLongestChain { - notOnLongestChain = append(notOnLongestChain, hash) + notOnLongestChain = append(notOnLongestChain, item.Hash) } } return notOnLongestChain, nil } + +// FilterTxsUnmined returns the subset of hashes whose unmined_since is set: the txs a +// MarkTransactionsOnLongestChain(true) would change. Hashes missing from the UTXO store +// are skipped. +func FilterTxsUnmined(ctx context.Context, logger ulogger.Logger, tSettings *settings.Settings, utxoStore txMetaBatchReader, hashes []chainhash.Hash) ([]chainhash.Hash, error) { + items, err := batchReadTxMeta(ctx, logger, tSettings, utxoStore, hashes, fields.UnminedSince) + if err != nil { + return nil, errors.NewProcessingError("[FilterTxsUnmined] error getting transactions from utxo store", err) + } + + unmined := make([]chainhash.Hash, 0, len(hashes)) + + for _, item := range items { + if item.Err != nil { + if errors.Is(item.Err, errors.ErrTxNotFound) { + continue + } + + return nil, errors.NewProcessingError("[FilterTxsUnmined] error getting transaction %s from utxo store", item.Hash.String(), item.Err) + } + + if item.Data != nil && item.Data.UnminedSince != 0 { + unmined = append(unmined, item.Hash) + } + } + + return unmined, nil +} + +// batchReadTxMeta reads the given fields of hashes in concurrent BatchDecorate batches. The +// per-item result, including a per-item error, is on each returned item. +func batchReadTxMeta(ctx context.Context, logger ulogger.Logger, tSettings *settings.Settings, utxoStore txMetaBatchReader, + hashes []chainhash.Hash, readFields ...fields.FieldName) ([]*utxo.UnresolvedMetaData, error) { + items := make([]*utxo.UnresolvedMetaData, len(hashes)) + for idx, hash := range hashes { + items[idx] = &utxo.UnresolvedMetaData{Hash: hash, Idx: idx} + } + + g, gCtx := errgroup.WithContext(ctx) + // SafeSetLimit panics loudly on a zero limit (MaxMinedRoutines and GetBatcherSize both 0) + // rather than letting g.Go deadlock on a zero-capacity semaphore. + util.SafeSetLimit(logger, g, max(tSettings.UtxoStore.MaxMinedRoutines, tSettings.UtxoStore.GetBatcherSize*2)) + + for start := 0; start < len(items); start += filterNotOnLongestChainBatchSize { + batch := items[start:min(start+filterNotOnLongestChainBatchSize, len(items))] + + g.Go(func() error { + return utxoStore.BatchDecorate(gCtx, batch, readFields...) + }) + } + + if err := g.Wait(); err != nil { + return nil, err + } + + return items, nil +} diff --git a/services/blockassembly/liveness_reset_test.go b/services/blockassembly/liveness_reset_test.go index 3176f7334d..5f357a2afd 100644 --- a/services/blockassembly/liveness_reset_test.go +++ b/services/blockassembly/liveness_reset_test.go @@ -87,12 +87,12 @@ type probeUtxoStore struct { scanIterator utxo.ConsistencyScanIterator } -func (s *probeUtxoStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (s *probeUtxoStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { if !onLongestChain && s.onMarkUnmined != nil { s.onMarkUnmined() } - return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) + return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) } // ScanInconsistentUnminedTxs stands in the synthetic scan when one is set. The diff --git a/services/blockassembly/reset_pruned_moveback_test.go b/services/blockassembly/reset_pruned_moveback_test.go index 1831a99e54..ca1d08a321 100644 --- a/services/blockassembly/reset_pruned_moveback_test.go +++ b/services/blockassembly/reset_pruned_moveback_test.go @@ -30,14 +30,14 @@ type markRecorderStore struct { itemErrs map[chainhash.Hash]error } -func (s *markRecorderStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (s *markRecorderStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { if !onLongestChain { s.mu.Lock() s.markedFalse = append(s.markedFalse, txHashes...) s.mu.Unlock() } - return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) + return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) } func (s *markRecorderStore) BatchDecorate(ctx context.Context, items []*utxo.UnresolvedMetaData, f ...fields.FieldName) error { diff --git a/services/blockassembly/subtreeprocessor/SubtreeProcessor.go b/services/blockassembly/subtreeprocessor/SubtreeProcessor.go index d3b8997c90..69dcf4ed14 100644 --- a/services/blockassembly/subtreeprocessor/SubtreeProcessor.go +++ b/services/blockassembly/subtreeprocessor/SubtreeProcessor.go @@ -1637,8 +1637,11 @@ func (stp *SubtreeProcessor) reset(blockHeader *model.BlockHeader, moveBackBlock // (WHERE unmined_since IS NOT NULL) — will not find them, and they will be silently dropped // from block assembly after the reset. // - // This mirrors exactly what reorgBlocks() does at lines 2665-2710 before it finalises a reorg. - // The call is idempotent: for transactions that already have unmined_since set, it is a no-op. + // reorgBlocks() no longer sweeps the whole assembly (issue 2051); block validation repairs + // an overtaken tip's txs when the best chain changes, asynchronously. This sweep stays as + // the catch-all for a reset that lands before that repair, since reset is rare and is + // where a gap drops txs from block assembly. For transactions that already have + // unmined_since set, it is a no-op. { currentSubtree := stp.currentSubtree.Load() subtreeNodeCount := len(currentSubtree.Nodes) @@ -5126,10 +5129,12 @@ func (stp *SubtreeProcessor) reorgBlocks(ctx context.Context, moveBackBlocks []* // writes a mempool tx with unmined_since set, and setTxMined only clears it for a block // on the longest chain, so a pure mempool tx needs no repair. The one way such a tx can // look mined without being in a moveBack block is a tip that was overtaken before block - // assembly ever moved onto it; setTxMined repairs that at the source - // (BlockValidation.unmarkIfOvertaken). Folding the whole mempool in here cost a batch - // write per mempool tx on every reorg — 380M records and ~8 minutes at scale, with - // block assembly intake stalled throughout. + // assembly ever moved onto it. Block validation repairs those when the best chain + // changes (services/blockvalidation/overtaken_tip_repair.go), asynchronously; reset() + // keeps its whole-assembly sweep as the catch-all for a reset that lands before that + // repair. Folding the whole mempool in here cost a batch write per mempool tx on every + // reorg — 380M records and ~8 minutes at scale, with block assembly intake stalled + // throughout. // // The set is restricted to txs still in block assembly, not the whole moveBack-minus- // moveForward set: MarkTransactionsOnLongestChain treats a missing record as fatal, and @@ -5332,10 +5337,13 @@ func (stp *SubtreeProcessor) markNotOnLongestChain(ctx context.Context, moveBack func (stp *SubtreeProcessor) checkMarkNotOnLongestChain(ctx context.Context, invalidBlock *model.Block, markNotOnLongestChain []chainhash.Hash) ([]chainhash.Hash, error) { stp.logger.Infof("[reorgBlocks] block %s we moved back is invalid, checking whether to mark transactions as not on longest chain", invalidBlock.String()) - // With defaults (MaxMinedRoutines=128) the limit is always >= 1. - checkMarkNotOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, stp.logger, stp.utxoStore, stp.blockchainClient, - invalidBlock.Header.HashPrevBlock, invalidBlock.ID, markNotOnLongestChain, - max(stp.settings.UtxoStore.MaxMinedRoutines, stp.settings.UtxoStore.GetBatcherSize*2)) + recentIDs, err := model.RecentChainBlockIDs(ctx, stp.blockchainClient, invalidBlock.Header.HashPrevBlock, 1000) + if err != nil { + return nil, errors.NewProcessingError("[reorgBlocks] error getting last block headers", err) + } + + checkMarkNotOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, stp.logger, stp.settings, stp.utxoStore, stp.blockchainClient, + recentIDs, invalidBlock.ID, markNotOnLongestChain, false) if err != nil { return nil, errors.NewProcessingError("[reorgBlocks] error filtering transactions not on longest chain", err) } diff --git a/services/blockassembly/subtreeprocessor/reorg_collect_all_txs_test.go b/services/blockassembly/subtreeprocessor/reorg_collect_all_txs_test.go index f982c57fc4..745da547cd 100644 --- a/services/blockassembly/subtreeprocessor/reorg_collect_all_txs_test.go +++ b/services/blockassembly/subtreeprocessor/reorg_collect_all_txs_test.go @@ -31,7 +31,7 @@ type markRecordingStore struct { markedOn []chainhash.Hash } -func (s *markRecordingStore) MarkTransactionsOnLongestChain(_ context.Context, hashes []chainhash.Hash, onLongestChain bool) error { +func (s *markRecordingStore) MarkTransactionsOnLongestChain(_ context.Context, hashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { if onLongestChain { s.mu.Lock() s.markedOn = append(s.markedOn, hashes...) diff --git a/services/blockassembly/subtreeprocessor/reorg_mark_partition_test.go b/services/blockassembly/subtreeprocessor/reorg_mark_partition_test.go index b8cf84285c..4a6d761bf6 100644 --- a/services/blockassembly/subtreeprocessor/reorg_mark_partition_test.go +++ b/services/blockassembly/subtreeprocessor/reorg_mark_partition_test.go @@ -255,6 +255,11 @@ func TestCheckMarkNotOnLongestChain(t *testing.T) { txOtherChain := createMinedTx(t, ctx, store, 2, otherChainID) // on current chain -> keep txForked := createMinedTx(t, ctx, store, 3, forkedChainID) // not on current chain -> mark false + // Only in the invalid block, but already unmined (block validation's unsetMined got there + // first): marking it is a no-op, so it must not be returned. + txAlreadyUnmined := createMinedTx(t, ctx, store, 4, invalidBlockID) + require.NoError(t, store.MarkTransactionsOnLongestChain(ctx, []chainhash.Hash{txAlreadyUnmined}, false)) + invalidBlock := &model.Block{ ID: invalidBlockID, Height: 1, @@ -268,7 +273,7 @@ func TestCheckMarkNotOnLongestChain(t *testing.T) { }, } - input := []chainhash.Hash{txOnlyInvalid, txRecent, txOtherChain, txForked} + input := []chainhash.Hash{txOnlyInvalid, txRecent, txOtherChain, txForked, txAlreadyUnmined} result, err := stp.checkMarkNotOnLongestChain(ctx, invalidBlock, input) require.NoError(t, err) diff --git a/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go index 30c21f76fa..30d208c6f8 100644 --- a/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go +++ b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go @@ -17,7 +17,6 @@ import ( blob_memory "github.com/bsv-blockchain/teranode/stores/blob/memory" utxostore "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/stores/utxo/fields" - "github.com/bsv-blockchain/teranode/stores/utxo/meta" "github.com/bsv-blockchain/teranode/stores/utxo/sql" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util/test" @@ -26,7 +25,7 @@ import ( ) // scopeMarkRecorder records the hashes of every MarkTransactionsOnLongestChain call, per -// direction, and of every Get, and then applies the call to the real store underneath. +// direction, and of every BatchDecorate read, and then applies the call to the real store underneath. type scopeMarkRecorder struct { utxostore.Store @@ -45,12 +44,14 @@ func newScopeMarkRecorder(store utxostore.Store) *scopeMarkRecorder { } } -func (s *scopeMarkRecorder) Get(ctx context.Context, hash *chainhash.Hash, fields ...fields.FieldName) (*meta.Data, error) { +func (s *scopeMarkRecorder) BatchDecorate(ctx context.Context, unresolvedMetaDataSlice []*utxostore.UnresolvedMetaData, fields ...fields.FieldName) error { s.mu.Lock() - s.gets[*hash] = struct{}{} + for _, item := range unresolvedMetaDataSlice { + s.gets[item.Hash] = struct{}{} + } s.mu.Unlock() - return s.Store.Get(ctx, hash, fields...) + return s.Store.BatchDecorate(ctx, unresolvedMetaDataSlice, fields...) } func (s *scopeMarkRecorder) gotten(hash chainhash.Hash) bool { @@ -62,7 +63,7 @@ func (s *scopeMarkRecorder) gotten(hash chainhash.Hash) bool { return ok } -func (s *scopeMarkRecorder) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (s *scopeMarkRecorder) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxostore.MarkLongestChainOption) error { s.mu.Lock() target := s.markFalse if onLongestChain { @@ -74,7 +75,7 @@ func (s *scopeMarkRecorder) MarkTransactionsOnLongestChain(ctx context.Context, } s.mu.Unlock() - return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) + return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) } func (s *scopeMarkRecorder) markedFalse(hash chainhash.Hash) bool { diff --git a/services/blockpersister/Server_test.go b/services/blockpersister/Server_test.go index eb2eaf54c9..da06f9ec7b 100644 --- a/services/blockpersister/Server_test.go +++ b/services/blockpersister/Server_test.go @@ -1071,7 +1071,7 @@ func (m *MockUTXOStore) CompleteConflictIntent(ctx context.Context, intentID cha func (m *MockUTXOStore) PendingConflictIntents(ctx context.Context) ([]utxo.ConflictIntent, error) { return nil, nil } -func (m *MockUTXOStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (m *MockUTXOStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { return nil } func (m *MockUTXOStore) SetBlockHeight(height uint32) error { return nil } diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 5ce2074c52..5feff3c091 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -295,6 +295,23 @@ type BlockValidation struct { // wait in checkParentProcessingComplete is unaffected by the backoff window. setMinedRetryPending sync.Map + // minedOnChain tracks the blocks setTxMined wrote as on the longest chain, for the + // overtaken-tip repair (overtaken_tip_repair.go), which the setMined worker runs a slice + // at a time. tipRepairPending requests a slice; tipRepairSignal (capacity 1) wakes an + // idle worker for it. tipRepairMu guards tipRepairSeeded and tipRepairJob. + minedOnChain *minedOnChainTracker + tipRepairPending atomic.Bool + tipRepairSignal chan struct{} + tipRepairMu sync.Mutex + tipRepairSeeded bool + tipRepairJob *tipRepairJob + // tipRepairMainChain caches the main-chain ids ending at tipRepairBestHash (guarded by + // tipRepairMu), so a best block that only extends the chain costs one call. + tipRepairMainChain map[uint32]bool + tipRepairBestHash chainhash.Hash + // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. + tipRepairRetry atomic.Bool + // revalidateBlockChan receives blocks that need revalidation revalidateBlockChan chan revalidateBlockData @@ -448,6 +465,8 @@ func NewBlockValidation(ctx context.Context, logger ulogger.Logger, tSettings *s setMinedChan: make(chan *chainhash.Hash, 1000), setMinedOverflow: make(map[chainhash.Hash]struct{}), setMinedOverflowSignal: make(chan struct{}, 1), + minedOnChain: newMinedOnChainTracker(), + tipRepairSignal: make(chan struct{}, 1), revalidateBlockChan: make(chan revalidateBlockData, 2), revalidateWorkerStopped: make(chan struct{}), stats: gocore.NewStat("blockvalidation"), @@ -579,6 +598,12 @@ func NewBlockValidation(ctx context.Context, logger ulogger.Logger, tSettings *s bv.enqueueSetMined(&cHash) } + // Any best-chain change may have overtaken a block whose txs setTxMined + // marked mined; let the overtaken-tip repair look. + if notification.Type == model.NotificationType_Block { + bv.signalTipRepair() + } + // Listen for BlockMinedUnset notifications (sent by InvalidateBlock RPC) // This triggers immediate processing instead of waiting for periodic job if notification.Type == model.NotificationType_BlockMinedUnset { @@ -636,6 +661,10 @@ func initialiseInvalidBlockKafkaProducer(ctx context.Context, logger ulogger.Log func (u *BlockValidation) start(ctx context.Context) error { g, gCtx := errgroup.WithContext(ctx) + // Read once here, with the rest of start's startup reads; the setMined worker below + // runs the overtaken-tip repair with this client. + blockchainClient := u.blockchainClient + if u.blockchainClient != nil { // check whether all old blocks have their subtrees_set set u.processSubtreesNotSet(gCtx, g) @@ -679,6 +708,10 @@ func (u *BlockValidation) start(ctx context.Context) error { case <-ticker.C: u.processSubtreesNotSet(ctx, g) u.processBlockMinedNotSet(ctx) + // Retries a failed overtaken-tip repair while the chain is idle. + if u.tipRepairRetry.Load() { + u.signalTipRepair() + } } } }() @@ -696,10 +729,22 @@ func (u *BlockValidation) start(ctx context.Context) error { defer u.logger.Infof("[BlockValidation:start] setMinedChan worker stopped") for { + // One slice of the overtaken-tip repair between blocks, when one is pending. + // Running it here keeps it from overlapping a setTxMined write; the slice size + // bounds how long it delays the next block. + if u.tipRepairPending.Swap(false) { + u.runTipRepairSlice(ctx, blockchainClient) + } + // nextSetMinedHash drains setMinedChan first and falls back to the - // overflow set; it returns nil only when ctx is done. + // overflow set; it returns nil when ctx is done or a repair slice was + // requested while it waited. blockHash := u.nextSetMinedHash(ctx) if blockHash == nil { + if ctx.Err() == nil { + continue + } + u.logger.Warnf("[BlockValidation:start] exiting setMined goroutine: %s", ctx.Err()) return @@ -1000,8 +1045,9 @@ func (u *BlockValidation) enqueueSetMined(blockHash *chainhash.Hash) { // nextSetMinedHash returns the next block hash for the setMined worker to process, // preferring setMinedChan and falling back to the overflow set. It blocks until a hash -// is available and returns nil only when ctx is done. Only the setMined worker calls -// this. +// is available, and returns nil when ctx is done or when an overtaken-tip repair step was +// requested while it waited, which the caller tells apart by ctx.Err(). Only the setMined +// worker calls this. func (u *BlockValidation) nextSetMinedHash(ctx context.Context) *chainhash.Hash { for { select { @@ -1021,6 +1067,8 @@ func (u *BlockValidation) nextSetMinedHash(ctx context.Context) *chainhash.Hash return blockHash case <-u.setMinedOverflowSignal: // re-check the overflow set + case <-u.tipRepairSignal: + return nil // the caller runs the requested repair slice } } } @@ -1579,6 +1627,13 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return errors.NewServiceError("[setTxMined][%s] failed to get block header ids", blockHash.String()) } + // Track before writing: if this write clears unmined_since and the block is later + // overtaken, the overtaken-tip repair must know about it, even if this attempt fails + // and a retry sees onLongestChain=false. + if onLongestChain && (len(unsetMined) == 0 || !unsetMined[0]) { + u.minedOnChain.track(*blockHash, blockHeaderMeta.ID, blockHeaderMeta.Height, false) + } + // add the transactions in this block to the block IDs in the utxo store if err = model.UpdateTxMinedStatus( ctx, @@ -1594,11 +1649,16 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain ); err != nil { // check whether we got already mined errors and mark the block as invalid if errors.Is(err, errors.ErrBlockInvalid) { + // unsetMined owns an invalid block's txs, not the overtaken-tip repair + u.minedOnChain.drop(*blockHash) + // mark the block as invalid in the blockchain // setTxMined no longer knows which peer supplied the block, so no // provenance is attached; the p2p consumer falls back to its peer map. return u.markBlockAsInvalid(ctx, block, "contains transactions already on our chain: "+err.Error(), "", "") } else if errors.Is(err, errors.ErrBlockParentNotMined) { + // Another call for this block is in flight. Its tracking stays: that write + // marks it written when it completes. u.logger.Warnf("[setTxMined][%s] skipping, already in progress of setMined for block", block.Hash().String()) return nil } @@ -1606,12 +1666,12 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return errors.NewProcessingError("[setTxMined][%s] error updating tx mined status", block.Hash().String(), err) } - // Must run before SetBlockMinedSet below: block assembly waits on mined_set before it - // acts on this block, so the repair is in place before anything can build on it. - if onLongestChain && (len(unsetMined) == 0 || !unsetMined[0]) { - if err = u.unmarkIfOvertaken(ctx, block, blockHeaderMeta.ID); err != nil { - return err - } + // The write is complete, whichever onLongestChain this attempt used: a block tracked by + // an earlier attempt (whose clears may have landed) becomes eligible for the repair now. + // Ask for a slice: a chain change that happened while the write was in flight would + // otherwise wait for the next one. + if u.minedOnChain.markWritten(*blockHash) { + u.signalTipRepair() } // Close mmap-backed subtrees and clear to free memory. Routed through @@ -1634,66 +1694,6 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return nil } -// unmarkIfOvertaken repairs the txs of a block that left the longest chain while its -// setTxMined was running. -// -// setTxMined decides onLongestChain once, before writing. If the block was the tip then, -// the write cleared unmined_since on its txs. If a competing chain overtook it before block -// assembly moved onto it, block assembly reaches the new tip by a pure catch-up, never sees -// this block as a moveBack block, and nothing else restores unmined_since. Those txs would -// then sit in the mempool looking mined: prunable, and invisible to loadUnminedTransactions. -// -// Only the txs not mined on the new longest chain are re-marked; a tx shared with the -// winning chain is mined there and keeps its cleared unmined_since. This is the source-side -// repair that lets block assembly's reorg mark only the txs a reorg actually moves -// (issue 2051). -func (u *BlockValidation) unmarkIfOvertaken(ctx context.Context, block *model.Block, blockID uint32) error { - stillOnLongestChain, err := u.blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{blockID}) - if err != nil { - return errors.NewServiceError("[setTxMined][%s] failed to re-check if block is on longest chain", block.Hash().String(), err) - } - - if stillOnLongestChain { - return nil - } - - bestBlockHeader, _, err := u.blockchainClient.GetBestBlockHeader(ctx) - if err != nil { - return errors.NewServiceError("[setTxMined][%s] failed to get best block header", block.Hash().String(), err) - } - - txHashes := make([]chainhash.Hash, 0, block.TransactionCount) - - for _, subtree := range block.SubtreeSlices { - if subtree == nil { - return errors.NewProcessingError("[setTxMined][%s] missing subtree while re-marking an overtaken block", block.Hash().String()) - } - - for _, node := range subtree.Nodes { - if !node.Hash.Equal(subtreepkg.CoinbasePlaceholderHashValue) { - txHashes = append(txHashes, node.Hash) - } - } - } - - notOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, u.logger, u.utxoStore, u.blockchainClient, - bestBlockHeader.Hash(), blockID, txHashes, - max(u.settings.UtxoStore.MaxMinedRoutines, u.settings.UtxoStore.GetBatcherSize*2)) - if err != nil { - return errors.NewProcessingError("[setTxMined][%s] error filtering txs of overtaken block", block.Hash().String(), err) - } - - u.logger.Warnf("[setTxMined][%s] block left the longest chain during setTxMined, marking %d of %d txs as not on longest chain", block.Hash().String(), len(notOnLongestChain), len(txHashes)) - - if len(notOnLongestChain) > 0 { - if err = u.utxoStore.MarkTransactionsOnLongestChain(ctx, notOnLongestChain, false); err != nil { - return errors.NewProcessingError("[setTxMined][%s] error marking txs of overtaken block as not on longest chain", block.Hash().String(), err) - } - } - - return nil -} - // isParentMined verifies if a block's parent has been successfully mined and committed to the blockchain. // // This function performs a critical validation step in the block processing pipeline by diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go new file mode 100644 index 0000000000..db044f89b6 --- /dev/null +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -0,0 +1,641 @@ +package blockvalidation + +import ( + "bufio" + "context" + "sync" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + subtreepkg "github.com/bsv-blockchain/go-subtree" + "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/model" + "github.com/bsv-blockchain/teranode/pkg/fileformat" + "github.com/bsv-blockchain/teranode/services/blockchain" + "github.com/bsv-blockchain/teranode/stores/utxo" +) + +// Overtaken-tip repair (issue 2051). +// +// setTxMined clears unmined_since on a block's txs when the block is on the longest chain +// at the time it runs. If a competing chain later overtakes that block before block +// assembly has moved onto it, block assembly reaches the new tip by a pure catch-up and +// never sees the block as a moveBack block, so nothing in block assembly restores +// unmined_since. Those txs then sit in the mempool looking mined: the pruner treats them +// (and their parents) as prunable, and loadUnminedTransactions cannot see them. At scale +// block assembly lags by minutes, so this is the normal case in a fork storm, not an edge. +// +// Block assembly's reorg used to cover this by re-marking its whole mempool on every reorg, +// a batch write per mempool tx. The repair here replaces that: it tracks the blocks +// setTxMined wrote as on-chain and, when one has left the chain, re-marks its txs that are +// not mined on the new chain. +// +// It runs on the setMined worker, between blocks, one slice of txs at a time. Running it +// there, rather than concurrently, is what makes it correct: no setTxMined write can land +// in the middle of a slice's read-decide-write, so for a tx the overtaken block shares with +// another block both orders end right. If the slice runs first, the tx is stamped and the +// other block's setTxMined clears it again; if that setTxMined ran first, the tx's block +// ids show it is mined on the longest chain and the slice leaves it alone. The slice size +// bounds how long a repair delays the next block's mined_set. +// +// Limitations. The repair is serialised with this process's setMined worker only: it +// assumes one block validation instance writes setTxMined for the chain. Its state is in +// memory. After a restart the tracker is rebuilt from the chain without knowing which +// blocks were re-marked, so a re-marked block that comes back onto the chain afterwards +// keeps those txs flagged unmined until block assembly's next unmined-transaction load +// fixes them (fixUnminedSinceInconsistencies); and fork blocks deeper than +// tipRepairSeedForkDepth at the restart are not repaired. + +const ( + // tipRepairWindow is how deep below the best block a tracked block is kept. A reorg + // deeper than this is not repaired here. + tipRepairWindow = 1000 + + // tipRepairSeedForkDepth bounds the fork blocks the startup seed looks at, measured from + // the best height. The seed cannot tell a fork block that was once the tip from one + // that never was, and each one costs a read per tx, so it stays shallow. + tipRepairSeedForkDepth = 10 + + // tipRepairSeedMaxBlocks caps the recent blocks the startup seed reads to find them. + tipRepairSeedMaxBlocks = 1000 + + // tipRepairMaxAttempts is how many times a block may fail permanently (its block or + // subtrees are gone) before it is dropped. Transient failures do not count. + tipRepairMaxAttempts = 10 + + // defaultTipRepairSliceSize is used when BlockValidation.TipRepairSliceSize is unset. + defaultTipRepairSliceSize = 250_000 +) + +// tipRepairExpired reports whether a tracked block at height has left the window. Each slice +// reads main-chain ids with GetBlockHeaderIDs(best, tipRepairWindow), which covers best down +// to best-tipRepairWindow+1; a block below that would look off chain while it is not. +func tipRepairExpired(height, bestHeight uint32) bool { + return height+tipRepairWindow <= bestHeight +} + +// isPermanentRepairError reports whether a repair error means the block can never be +// repaired: the block or its subtrees are gone. +func isPermanentRepairError(err error) bool { + return errors.Is(err, errors.ErrBlockNotFound) || errors.Is(err, errors.ErrSubtreeNotFound) || + errors.Is(err, errors.ErrBlobNotFound) || errors.Is(err, errors.ErrNotFound) +} + +// minedOnChainBlock is a block setTxMined wrote with onLongestChain=true. +type minedOnChainBlock struct { + id uint32 + height uint32 + // gen changes whenever a new on-chain write of the block starts. A repair in progress + // for an older gen is abandoned: the new write cleared the txs again. + gen uint64 + // written is set once the block's setTxMined write has completed. A repair only starts + // on a written block. + written bool + // stamped is set once any of the block's txs were re-marked not on the longest chain, + // so they need marking mined again if the block comes back onto the chain. + stamped bool + // done is set when every tx of the block has been through the repair while it was off + // the chain; it is cleared when the block is seen back on the chain. + done bool + failures int // permanent failures so far +} + +// minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker +// tracks nothing, for a BlockValidation not built by NewBlockValidation. +type minedOnChainTracker struct { + mu sync.Mutex + blocks map[chainhash.Hash]minedOnChainBlock + nextGen uint64 +} + +func newMinedOnChainTracker() *minedOnChainTracker { + return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock)} +} + +// track watches a block. With written=false a new on-chain write is starting: it clears the +// block's txs again, so the block gets a new gen and starts over. With written=true (the +// restart seed) a block already watched keeps its state. +func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32, written bool) { + if t == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + b, ok := t.blocks[hash] + if ok && written { + return + } + + t.nextGen++ + t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, written: written, failures: b.failures} +} + +// markWritten records that the block's setTxMined write has completed and reports whether +// the block is tracked, in which case the caller asks for a repair slice. +func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { + if t == nil { + return false + } + + t.mu.Lock() + defer t.mu.Unlock() + + b, ok := t.blocks[hash] + if !ok { + return false + } + + b.written = true + t.blocks[hash] = b + + return true +} + +// update applies fn to the block's entry if it is still at gen. +func (t *minedOnChainTracker) update(hash chainhash.Hash, gen uint64, fn func(*minedOnChainBlock)) { + if t == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + if b, ok := t.blocks[hash]; ok && b.gen == gen { + fn(&b) + t.blocks[hash] = b + } +} + +func (t *minedOnChainTracker) drop(hash chainhash.Hash) { + if t == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + delete(t.blocks, hash) +} + +func (t *minedOnChainTracker) empty() bool { + if t == nil { + return true + } + + t.mu.Lock() + defer t.mu.Unlock() + + return len(t.blocks) == 0 +} + +func (t *minedOnChainTracker) snapshot() map[chainhash.Hash]minedOnChainBlock { + if t == nil { + return nil + } + + t.mu.Lock() + defer t.mu.Unlock() + + out := make(map[chainhash.Hash]minedOnChainBlock, len(t.blocks)) + for hash, b := range t.blocks { + out[hash] = b + } + + return out +} + +// tipRepairJobKind is what a tipRepairJob writes. +type tipRepairJobKind int + +const ( + // unmarkJob re-marks the txs of a block that left the chain and are not mined on it. + unmarkJob tipRepairJobKind = iota + // remineJob marks the txs of a re-marked block mined again, now that it is back on the + // chain. Every one of them carries the block's id, so clearing unmined_since is right + // for all of them. + remineJob +) + +// tipRepairJob is a repair in progress over one block, worked through one subtree, and +// within a subtree one slice of txs, per step. Only the current subtree's tx hashes are +// held. +type tipRepairJob struct { + kind tipRepairJobKind + hash chainhash.Hash + id uint32 + gen uint64 + subtrees []*chainhash.Hash + // nextSubtree is the index of the subtree to load once current is used up. + nextSubtree int + current []subtreepkg.Node + marked int + total int +} + +// signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending +// request covers any number of calls. +func (u *BlockValidation) signalTipRepair() { + u.tipRepairPending.Store(true) + + select { + case u.tipRepairSignal <- struct{}{}: + default: + } +} + +// runTipRepairSlice is called by the setMined worker when a slice was requested. It runs one +// slice and requests the next while there is more to do. A transient failure is retried on +// the next request (a chain change, a write, or the periodic tick, which keeps asking while +// tipRepairRetry is set). +func (u *BlockValidation) runTipRepairSlice(ctx context.Context, blockchainClient blockchain.ClientI) { + more, err := u.tipRepairSlice(ctx, blockchainClient) + u.tipRepairRetry.Store(err != nil) + + if err != nil { + if ctx.Err() != nil { + return + } + + u.logger.Errorf("[repairOvertakenTips] slice failed, will retry: %v", err) + + return + } + + if more { + u.signalTipRepair() + } +} + +// seedMinedOnChainTrackerLocked rebuilds the in-memory tracker after a restart: the +// main-chain blocks inside the window that are mined, and the mined fork blocks within +// tipRepairSeedForkDepth of the best height, which may have been the tip before the restart. +func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { + bestHeader, bestMeta, err := blockchainClient.GetBestBlockHeader(ctx) + if err != nil { + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get best block header", err) + } + + headers, metas, err := blockchainClient.GetBlockHeaders(ctx, bestHeader.Hash(), tipRepairWindow) + if err != nil { + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get main chain headers", err) + } + + u.trackMinedHeaders(headers, metas) + + // Fork blocks near the tip: one query for the highest blocks, side chains included. + // GetChainTips would do too, but on SQLite it holds a row cursor open across nested + // queries and deadlocks against a concurrent write transaction. + recent, err := blockchainClient.GetLastNBlocks(ctx, tipRepairSeedMaxBlocks, true, 0) + if err != nil { + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get recent blocks", err) + } + + for _, info := range recent { + if info.Height+tipRepairSeedForkDepth < bestMeta.Height { + break // highest first + } + + header, err := model.NewBlockHeaderFromBytes(info.BlockHeader) + if err != nil { + return errors.NewProcessingError("[repairOvertakenTips] seed: invalid block header at height %d", info.Height, err) + } + + _, meta, err := blockchainClient.GetBlockHeader(ctx, header.Hash()) + if err != nil { + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get block header %s", header.Hash().String(), err) + } + + u.trackMinedHeaders([]*model.BlockHeader{header}, []*model.BlockHeaderMeta{meta}) + } + + u.tipRepairSeeded = true + + return nil +} + +func (u *BlockValidation) trackMinedHeaders(headers []*model.BlockHeader, metas []*model.BlockHeaderMeta) { + for i, meta := range metas { + if i >= len(headers) || meta == nil || !meta.MinedSet || meta.Invalid { + continue + } + + u.minedOnChain.track(*headers[i].Hash(), meta.ID, meta.Height, true) + } +} + +// tipRepairSlice does one bounded step of the repair against the current best chain and +// reports whether more steps are pending. A step is one of: load the next subtree of the +// job in progress, or write the next slice of its txs; start a job, marking a re-marked +// block that is back on the chain mined again before repairing a block that left it. +// Housekeeping drops expired and invalid blocks. +// +// The repair is serialised with this process's setMined worker only. Several block +// validation instances writing setTxMined for the same chain would each run their own +// repair, unserialised against the others' writes. +func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient blockchain.ClientI) (more bool, err error) { + u.tipRepairMu.Lock() + defer u.tipRepairMu.Unlock() + + // Nothing to do until this process has written a block as on the longest chain. The + // restart seed waits for that too: after a restart the next block's setTxMined comes + // within one block interval, and a process that never marks blocks mined (or a test + // harness that never runs setTxMined) makes no blockchain calls here at all. + if !u.tipRepairSeeded && u.minedOnChain.empty() { + return false, nil + } + + if !u.tipRepairSeeded { + if err = u.seedMinedOnChainTrackerLocked(ctx, blockchainClient); err != nil { + return false, err + } + } + + onMainChain, bestHeight, err := u.mainChainIDsLocked(ctx, blockchainClient) + if err != nil { + return false, err + } + + tracked := u.minedOnChain.snapshot() + + for hash, b := range tracked { + switch { + case tipRepairExpired(b.height, bestHeight): + if b.written && !onMainChain[b.id] && !b.done { + u.logger.Warnf("[repairOvertakenTips][%s] block left the repair window before its repair finished; some of its txs may still look mined", hash.String()) + } + + u.minedOnChain.drop(hash) + delete(tracked, hash) + + case onMainChain[b.id] && b.done && !b.stamped: + // Back on the chain and nothing was re-marked: nothing to undo, but if it + // leaves again it needs repairing again. + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.done = false }) + b.done = false + tracked[hash] = b + } + } + + if job := u.tipRepairJob; job != nil { + b, ok := tracked[job.hash] + + switch { + case !ok || b.gen != job.gen: + u.tipRepairJob = nil // dropped, or rewritten by a new setTxMined + return true, nil + case job.kind == unmarkJob && onMainChain[b.id], job.kind == remineJob && !onMainChain[b.id]: + u.tipRepairJob = nil // the chain moved; the next step starts the right job + return true, nil + } + + return true, u.jobStep(ctx, blockchainClient, job, onMainChain) + } + + // A re-marked block back on the chain first: its txs look unmined while mined. + for hash, b := range tracked { + if b.stamped && onMainChain[b.id] { + return true, u.startJob(ctx, blockchainClient, remineJob, hash, b) + } + } + + for hash, b := range tracked { + if !b.written || b.done || onMainChain[b.id] { + continue + } + + _, meta, err := blockchainClient.GetBlockHeader(ctx, &hash) + if err != nil { + return true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block header", hash.String(), err)) + } + + if meta == nil { + return true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] block header meta is nil", hash.String())) + } + + if meta.Invalid { + // block validation's unsetMined owns invalid blocks + u.minedOnChain.drop(hash) + continue + } + + return true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) + } + + return false, nil +} + +// mainChainIDsLocked returns the ids of the tipRepairWindow blocks ending at the best block, +// and the best height. When the best block only extends the one seen last time, the cached +// set is extended with the new block's id instead of fetched again, so a step costs two +// small blockchain calls in steady state rather than a 1000-id fetch. +func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClient blockchain.ClientI) (map[uint32]bool, uint32, error) { + bestHeader, bestMeta, err := blockchainClient.GetBestBlockHeader(ctx) + if err != nil { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get best block header", err) + } + + bestHash := *bestHeader.Hash() + + switch { + case u.tipRepairMainChain != nil && bestHash == u.tipRepairBestHash: + return u.tipRepairMainChain, bestMeta.Height, nil + + case u.tipRepairMainChain != nil && bestHeader.HashPrevBlock.IsEqual(&u.tipRepairBestHash): + // GetBestBlockHeader's meta carries no block id over gRPC, so ask for it. + ids, err := blockchainClient.GetBlockHeaderIDs(ctx, &bestHash, 1) + if err != nil || len(ids) == 0 { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get best block id", err) + } + + // Ids that fell out of the window stay in the set; tipRepairExpired drops their + // blocks before the set is consulted for them. + u.tipRepairMainChain[ids[0]] = true + u.tipRepairBestHash = bestHash + + return u.tipRepairMainChain, bestMeta.Height, nil + } + + ids, err := blockchainClient.GetBlockHeaderIDs(ctx, &bestHash, tipRepairWindow) + if err != nil { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get main chain block ids", err) + } + + onMainChain := make(map[uint32]bool, len(ids)) + for _, id := range ids { + onMainChain[id] = true + } + + u.tipRepairMainChain = onMainChain + u.tipRepairBestHash = bestHash + + return onMainChain, bestMeta.Height, nil +} + +// startJob begins a job over the block: it reads the block's subtree list, not its txs. +func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock) error { + block, err := blockchainClient.GetBlock(ctx, &hash) + if err != nil { + return u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block", hash.String(), err)) + } + + if kind == remineJob { + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.done = false }) + } + + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: block.Subtrees} + + return nil +} + +// jobStep writes the next slice of the job's txs, first loading the next subtree if the +// current one is used up, and finishes the job with its last slice. A subtree that is gone +// or unreadable is skipped with an error log, so the rest of the block is still repaired. +func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { + for len(job.current) == 0 { + if job.nextSubtree >= len(job.subtrees) { + u.finishJob(job) + return nil + } + + subtreeHash := job.subtrees[job.nextSubtree] + + nodes, err := u.loadSubtreeNodes(ctx, subtreeHash) + if err != nil { + if !isPermanentRepairError(err) { + return errors.NewProcessingError("[repairOvertakenTips][%s] failed to load subtree %d", job.hash.String(), job.nextSubtree, err) + } + + u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) + } + + job.nextSubtree++ + job.current = nodes + } + + sliceSize := u.settings.BlockValidation.TipRepairSliceSize + if sliceSize <= 0 { + sliceSize = defaultTipRepairSliceSize + } + + sliceNodes := job.current[:min(sliceSize, len(job.current))] + + slice := make([]chainhash.Hash, 0, len(sliceNodes)) + for _, node := range sliceNodes { + if !node.Hash.Equal(subtreepkg.CoinbasePlaceholderHashValue) { + slice = append(slice, node.Hash) + } + } + + switch job.kind { + case unmarkJob: + notOnLongestChain, err := model.FilterTxsNotOnLongestChain(ctx, u.logger, u.settings, u.utxoStore, blockchainClient, onMainChain, job.id, slice, true) + if err != nil { + return errors.NewProcessingError("[repairOvertakenTips][%s] error filtering txs of overtaken block", job.hash.String(), err) + } + + if len(notOnLongestChain) > 0 { + // Before writing, so a write that fails part way still counts as stamped. + u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.stamped = true }) + + if err = u.utxoStore.MarkTransactionsOnLongestChain(ctx, notOnLongestChain, false, utxo.WithSkipMissing()); err != nil { + return errors.NewProcessingError("[repairOvertakenTips][%s] error marking txs of overtaken block as not on longest chain", job.hash.String(), err) + } + } + + job.marked += len(notOnLongestChain) + + case remineJob: + // Only the txs that were stamped need writing. + unmined, err := model.FilterTxsUnmined(ctx, u.logger, u.settings, u.utxoStore, slice) + if err != nil { + return errors.NewProcessingError("[repairOvertakenTips][%s] error reading txs of block back on the longest chain", job.hash.String(), err) + } + + if len(unmined) > 0 { + if err = u.utxoStore.MarkTransactionsOnLongestChain(ctx, unmined, true, utxo.WithSkipMissing()); err != nil { + return errors.NewProcessingError("[repairOvertakenTips][%s] error marking txs of block back on the longest chain", job.hash.String(), err) + } + } + + job.marked += len(unmined) + } + + job.total += len(slice) + job.current = job.current[len(sliceNodes):] + + if len(job.current) == 0 && job.nextSubtree >= len(job.subtrees) { + u.finishJob(job) + } + + return nil +} + +// finishJob records a completed job on its block. +func (u *BlockValidation) finishJob(job *tipRepairJob) { + u.tipRepairJob = nil + + switch job.kind { + case unmarkJob: + u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { + e.done = true + e.failures = 0 + }) + + u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain after its txs were marked mined, marked %d of %d txs as not on longest chain", job.hash.String(), job.marked, job.total) + + case remineJob: + u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { + e.stamped = false + e.failures = 0 + }) + + u.logger.Infof("[repairOvertakenTips][%s] block is back on the longest chain, marked %d of %d txs mined again", job.hash.String(), job.marked, job.total) + } +} + +// repairFailed records a failed step for a block and returns err. A permanent failure (the +// block or its subtrees are gone) counts toward tipRepairMaxAttempts, after which the block +// is dropped; a transient one is only retried. +func (u *BlockValidation) repairFailed(hash chainhash.Hash, gen uint64, err error) error { + if !isPermanentRepairError(err) { + return err + } + + failures := 0 + u.minedOnChain.update(hash, gen, func(e *minedOnChainBlock) { + e.failures++ + failures = e.failures + }) + + if failures >= tipRepairMaxAttempts { + u.logger.Errorf("[repairOvertakenTips][%s] giving up after %d failed attempts; its txs may still look mined: %v", hash.String(), tipRepairMaxAttempts, err) + u.minedOnChain.drop(hash) + + if u.tipRepairJob != nil && u.tipRepairJob.hash == hash { + u.tipRepairJob = nil + } + } + + return err +} + +// loadSubtreeNodes returns the nodes of one subtree. +func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { + reader, err := u.subtreeStore.GetIoReader(ctx, subtreeHash[:], fileformat.FileTypeSubtree) + if err != nil { + return nil, err + } + + defer func() { + _ = reader.Close() + }() + + subtree, err := subtreepkg.NewSubtreeFromReader(bufio.NewReader(reader)) + if err != nil { + // A corrupt subtree cannot be read on a retry either. + return nil, errors.NewSubtreeNotFoundError("failed to deserialize subtree %s", subtreeHash.String(), err) + } + + return subtree.Nodes, nil +} diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index c143d0b35c..4a894d9e07 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -3,7 +3,9 @@ package blockvalidation import ( "context" "net/url" + "slices" "sync" + "sync/atomic" "testing" "time" @@ -11,10 +13,15 @@ import ( "github.com/bsv-blockchain/go-bt/v2/bscript" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" + "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/model" "github.com/bsv-blockchain/teranode/pkg/fileformat" "github.com/bsv-blockchain/teranode/services/blockchain" + "github.com/bsv-blockchain/teranode/services/subtreevalidation" + "github.com/bsv-blockchain/teranode/settings" + "github.com/bsv-blockchain/teranode/stores/blob" blockchain_store "github.com/bsv-blockchain/teranode/stores/blockchain" + blockchainoptions "github.com/bsv-blockchain/teranode/stores/blockchain/options" utxostore "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/stores/utxo/fields" "github.com/bsv-blockchain/teranode/ulogger" @@ -22,25 +29,45 @@ import ( "github.com/stretchr/testify/require" ) -// overtakeOnSetMined runs fn once, right after the first SetMinedMulti call for blockID -// has been applied. It lets a test move the chain tip while setTxMined is in flight, -// which is the window between setTxMined's onLongestChain check and its completion. -type overtakeOnSetMined struct { +// overtakeTestStore wraps the UTXO store so a test can act at precise points: +// afterSetMined runs once, right after the first SetMinedMulti for blockID, and +// failMarkFalse makes the next N MarkTransactionsOnLongestChain(false) calls fail. +type overtakeTestStore struct { utxostore.Store - blockID uint32 - once sync.Once - fn func() + + blockID uint32 + once sync.Once + afterSetMined func() + failMarkFalse atomic.Int32 + + markedTrueMu sync.Mutex + markedTrue []chainhash.Hash // every hash passed to MarkTransactionsOnLongestChain(true) } -func (s *overtakeOnSetMined) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { +func (s *overtakeTestStore) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { res, err := s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) - if err == nil && minedBlockInfo.BlockID == s.blockID { - s.once.Do(s.fn) + if err == nil && s.afterSetMined != nil && minedBlockInfo.BlockID == s.blockID { + s.once.Do(s.afterSetMined) } return res, err } +func (s *overtakeTestStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxostore.MarkLongestChainOption) error { + if onLongestChain { + s.markedTrueMu.Lock() + s.markedTrue = append(s.markedTrue, txHashes...) + s.markedTrueMu.Unlock() + } + + if !onLongestChain && s.failMarkFalse.Load() > 0 { + s.failMarkFalse.Add(-1) + return errors.NewStorageError("injected mark failure") + } + + return s.Store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) +} + // coinbaseOnlyBlock builds a mined block with no subtrees on parent. tag makes the // coinbase, and so the block, unique. func coinbaseOnlyBlock(t *testing.T, parent *chainhash.Hash, nBits *model.NBit, height uint32, tag byte) *model.Block { @@ -59,25 +86,32 @@ func coinbaseOnlyBlock(t *testing.T, parent *chainhash.Hash, nBits *model.NBit, return block } -// TestSetTxMinedStatus_BlockOvertakenDuringSetMined pins the repair for a tip that is -// overtaken while its setTxMined runs (issue 2051). -// -// setTxMined decides onLongestChain once, before it writes. If block B is the tip at that -// point, the write clears unmined_since on B's txs. If a competing chain then overtakes B -// before block assembly ever moves onto B, block assembly reaches the new tip by a pure -// catch-up and never sees B as a moveBack block, so nothing restores unmined_since on B's -// txs. They stay in block assembly's mempool looking mined, which makes them prunable and -// invisible to loadUnminedTransactions. setTxMined must notice B fell off the chain and -// re-mark the txs that are not mined on the new chain. A tx that B shares with the winning -// chain is mined there and must stay cleared. -func TestSetTxMinedStatus_BlockOvertakenDuringSetMined(t *testing.T) { +// overtakeFixture is a chain with block B on genesis mining two mempool txs: +// bOnly (only ever in B) and shared (also mined by the competing chain). +type overtakeFixture struct { + ctx context.Context + tSettings *settings.Settings + store *overtakeTestStore + blockchainClient blockchain.ClientI + subtreeStore blob.Store + txStore blob.Store + subtreeValidationClient subtreevalidation.Interface + nBits *model.NBit + blockB *model.Block + blockBID uint32 + bOnly *chainhash.Hash + shared *chainhash.Hash +} + +func newOvertakeFixture(t *testing.T) *overtakeFixture { + t.Helper() initPrometheusMetrics() ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + t.Cleanup(cancel) innerUtxoStore, subtreeValidationClient, _, txStore, subtreeStore, deferFunc := setup(t) - defer deferFunc() + t.Cleanup(deferFunc) tSettings := test.CreateBaseTestSettings(t) blockChainStore, err := blockchain_store.NewStore(ulogger.TestLogger{}, &url.URL{Scheme: "sqlitememory"}, tSettings) @@ -95,13 +129,6 @@ func TestSetTxMinedStatus_BlockOvertakenDuringSetMined(t *testing.T) { _, _, err = innerUtxoStore.SpendAndCreate(ctx, sharedTx, 1, utxostore.WithCreateOnly()) require.NoError(t, err) - for _, h := range []*chainhash.Hash{bOnlyTx.TxIDChainHash(), sharedTx.TxIDChainHash()} { - m, err := innerUtxoStore.Get(ctx, h, fields.UnminedSince) - require.NoError(t, err) - require.NotZero(t, m.UnminedSince, "precondition: mempool tx is created unmined") - } - - // Block B on genesis: coinbase, bOnly, shared. subtree, err := subtreepkg.NewTreeByLeafCount(4) require.NoError(t, err) require.NoError(t, subtree.AddCoinbaseNode()) @@ -123,109 +150,616 @@ func TestSetTxMinedStatus_BlockOvertakenDuringSetMined(t *testing.T) { blockB, err := model.NewBlock(headerB, coinbaseB, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), totalSize, 1, 0) //nolint:gosec require.NoError(t, err) - require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test")) + require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test", blockchainoptions.WithSubtreesSet(true))) _, metaB, err := blockchainClient.GetBlockHeader(ctx, blockB.Hash()) require.NoError(t, err) - onChain, err := blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{metaB.ID}) - require.NoError(t, err) - require.True(t, onChain, "precondition: B is the tip when setTxMined starts") + return &overtakeFixture{ + ctx: ctx, + tSettings: tSettings, + store: &overtakeTestStore{Store: innerUtxoStore, blockID: metaB.ID}, + blockchainClient: blockchainClient, + subtreeStore: subtreeStore, + txStore: txStore, + subtreeValidationClient: subtreeValidationClient, + nBits: nBits, + blockB: blockB, + blockBID: metaB.ID, + bOnly: bOnlyTx.TxIDChainHash(), + shared: sharedTx.TxIDChainHash(), + } +} - // The competing chain B' -> C' on genesis, which overtakes B mid-setTxMined. B' also - // mines sharedTx, so sharedTx is mined on the winning chain. - blockB2 := coinbaseOnlyBlock(t, tSettings.ChainCfgParams.GenesisHash, nBits, 1, 0xb2) - blockC2 := coinbaseOnlyBlock(t, blockB2.Hash(), nBits, 2, 0xc2) +// newBlockValidation returns a BlockValidation whose background workers have stopped, so +// the test drives every repair slice itself (drainTipRepair) and sees them in order. +func (f *overtakeFixture) newBlockValidation(t *testing.T) *BlockValidation { + t.Helper() - utxoStore := &overtakeOnSetMined{ - Store: innerUtxoStore, - blockID: metaB.ID, - fn: func() { - // Header timestamps must not decrease along the chain. - time.Sleep(10 * time.Millisecond) + ctx, stop := context.WithCancel(f.ctx) + bv := f.newRunningBlockValidation(ctx) + stop() + bv.backgroundTasks.Wait() - require.NoError(t, blockchainClient.AddBlock(ctx, blockB2, "test")) - require.NoError(t, blockchainClient.AddBlock(ctx, blockC2, "test")) + return bv +} - _, metaB2, err := blockchainClient.GetBlockHeader(ctx, blockB2.Hash()) - require.NoError(t, err) +// newRunningBlockValidation returns a BlockValidation with its background workers running: +// the setMined worker runs the repair on its own, as in production. +func (f *overtakeFixture) newRunningBlockValidation(ctx context.Context) *BlockValidation { + return NewBlockValidation(ctx, ulogger.TestLogger{}, f.tSettings, f.blockchainClient, f.subtreeStore, f.txStore, f.store, nil, f.subtreeValidationClient) +} - _, err = innerUtxoStore.SetMinedMulti(ctx, []*chainhash.Hash{sharedTx.TxIDChainHash()}, utxostore.MinedBlockInfo{ - BlockID: metaB2.ID, - BlockHeight: 1, - SubtreeIdx: 0, - OnLongestChain: true, - }) - require.NoError(t, err) - }, +// drainTipRepair runs repair slices until none is pending, as the setMined worker would +// between blocks, and returns the number of slices. It stops at the first error. +func drainTipRepair(t *testing.T, bv *BlockValidation) (int, error) { + t.Helper() + + for slices := 1; slices < 1000; slices++ { + more, err := bv.tipRepairSlice(context.Background(), bv.blockchainClient) + if err != nil || !more { + return slices, err + } } - blockValidation := NewBlockValidation(ctx, ulogger.TestLogger{}, tSettings, blockchainClient, subtreeStore, txStore, utxoStore, nil, subtreeValidationClient) + t.Fatal("repair did not settle") + + return 0, nil +} + +// overtakeB adds the competing chain B' -> C' on genesis, which overtakes B, and returns C'. +// B' also mines shared, so shared is mined on the winning chain. +func (f *overtakeFixture) overtakeB(t *testing.T) *model.Block { + t.Helper() + + blockC2, blockB2ID := f.overtakeBPendingSharedWrite(t) + f.mineSharedIn(t, blockB2ID) + + return blockC2 +} + +// mineSharedIn writes shared as mined on the longest chain in blockID: the competing +// block's own setTxMined. +func (f *overtakeFixture) mineSharedIn(t *testing.T, blockID uint32) { + t.Helper() + + _, err := f.store.Store.SetMinedMulti(f.ctx, []*chainhash.Hash{f.shared}, utxostore.MinedBlockInfo{ + BlockID: blockID, + BlockHeight: 1, + SubtreeIdx: 0, + OnLongestChain: true, + }) + require.NoError(t, err) +} + +// mineIn writes hash as mined on the longest chain in blockID. +func (f *overtakeFixture) mineIn(t *testing.T, hash *chainhash.Hash, blockID uint32) { + t.Helper() + + _, err := f.store.Store.SetMinedMulti(f.ctx, []*chainhash.Hash{hash}, utxostore.MinedBlockInfo{ + BlockID: blockID, + BlockHeight: 1, + SubtreeIdx: 0, + OnLongestChain: true, + }) + require.NoError(t, err) +} + +// overtakeBPendingSharedWrite is overtakeB before B' has written shared as mined. It returns +// C' and B”s id. +func (f *overtakeFixture) overtakeBPendingSharedWrite(t *testing.T) (*model.Block, uint32) { + t.Helper() + + // Header timestamps must not decrease along the chain. + time.Sleep(10 * time.Millisecond) + + blockB2 := coinbaseOnlyBlock(t, f.tSettings.ChainCfgParams.GenesisHash, f.nBits, 1, 0xb2) + blockC2 := coinbaseOnlyBlock(t, blockB2.Hash(), f.nBits, 2, 0xc2) + + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockB2, "test", blockchainoptions.WithSubtreesSet(true))) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockC2, "test", blockchainoptions.WithSubtreesSet(true))) + + _, metaB2, err := f.blockchainClient.GetBlockHeader(f.ctx, blockB2.Hash()) + require.NoError(t, err) + + require.False(t, f.onChain(t, f.blockBID), "precondition: the competing chain overtook B") + + return blockC2, metaB2.ID +} + +// extendB makes B's chain the longest again: B -> D2 -> D3 outworks B' -> C'. +func (f *overtakeFixture) extendB(t *testing.T) { + t.Helper() + + time.Sleep(10 * time.Millisecond) + + blockD2 := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xd2) + blockD3 := coinbaseOnlyBlock(t, blockD2.Hash(), f.nBits, 3, 0xd3) + + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD2, "test", blockchainoptions.WithSubtreesSet(true))) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD3, "test", blockchainoptions.WithSubtreesSet(true))) + + require.True(t, f.onChain(t, f.blockBID), "precondition: B is back on the longest chain") +} + +func (f *overtakeFixture) onChain(t *testing.T, blockID uint32) bool { + t.Helper() + + onChain, err := f.blockchainClient.CheckBlockIsInCurrentChain(f.ctx, []uint32{blockID}) + require.NoError(t, err) + + return onChain +} + +func (f *overtakeFixture) unminedSince(t *testing.T, hash *chainhash.Hash) uint32 { + t.Helper() + + m, err := f.store.Store.Get(f.ctx, hash, fields.UnminedSince) + require.NoError(t, err) - require.NoError(t, blockValidation.setTxMinedStatus(ctx, blockB.Hash())) + return m.UnminedSince +} - onChain, err = blockchainClient.CheckBlockIsInCurrentChain(ctx, []uint32{metaB.ID}) +// TestRepairOvertakenTips_OvertakenAfterSetMined pins the repair for a tip overtaken after +// its setTxMined finished (issue 2051). +// +// setTxMined clears unmined_since on the txs of the block that is the tip when it runs. If +// a competing chain overtakes that block before block assembly moves onto it, block +// assembly reaches the new tip by a pure catch-up, never sees the block as a moveBack +// block, and nothing in block assembly restores unmined_since. At scale block assembly lags +// by minutes, so this is the common case in a fork storm. The txs would sit in the mempool +// looking mined: prunable, and invisible to loadUnminedTransactions. The repair must +// re-mark the txs not mined on the new chain, and leave a tx the winning chain mined. +func TestRepairOvertakenTips_OvertakenAfterSetMined(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.True(t, f.onChain(t, f.blockBID), "precondition: B is the tip when setTxMined runs") + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: setTxMined marked B's txs mined") + + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) require.NoError(t, err) - require.False(t, onChain, "precondition: the competing chain overtook B during setTxMined") - bOnlyMeta, err := innerUtxoStore.Get(ctx, bOnlyTx.TxIDChainHash(), fields.UnminedSince, fields.BlockIDs) + require.NotZero(t, f.unminedSince(t, f.bOnly), "a tx mined only in the overtaken block must be unmined again") + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") +} + +// TestRepairOvertakenTips_OvertakenDuringSetMined covers the overtake landing while B's own +// setTxMined is running: the write still uses onLongestChain=true, so B must be tracked. +func TestRepairOvertakenTips_OvertakenDuringSetMined(t *testing.T) { + f := newOvertakeFixture(t) + f.store.afterSetMined = func() { _ = f.overtakeB(t) } + bv := f.newBlockValidation(t) + + require.True(t, f.onChain(t, f.blockBID), "precondition: B is the tip when setTxMined starts") + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + + bOnlyMeta, err := f.store.Store.Get(f.ctx, f.bOnly, fields.UnminedSince, fields.BlockIDs) require.NoError(t, err) - require.Equal(t, []uint32{metaB.ID}, bOnlyMeta.BlockIDs) + require.Equal(t, []uint32{f.blockBID}, bOnlyMeta.BlockIDs) require.NotZero(t, bOnlyMeta.UnminedSince, "a tx mined only in the overtaken block must be unmined again") + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") +} + +// TestRepairOvertakenTips_DuplicateSetMinedKeepsTracking: a second setTxMined for B while +// B's write is in flight is turned away as a duplicate. That must not stop B being tracked: +// the write in flight is what clears its txs. +func TestRepairOvertakenTips_DuplicateSetMinedKeepsTracking(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) - sharedMeta, err := innerUtxoStore.Get(ctx, sharedTx.TxIDChainHash(), fields.UnminedSince, fields.BlockIDs) + f.store.afterSetMined = func() { + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash()), "the duplicate is skipped, not an error") + } + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) require.NoError(t, err) - require.Zero(t, sharedMeta.UnminedSince, "a tx also mined on the winning chain must stay mined") + require.NotZero(t, f.unminedSince(t, f.bOnly), "B was still tracked, so it is repaired") } -// TestSetTxMinedStatus_BlockStillOnChain_NoRemark is the control: when B is still the tip -// after setTxMined writes, its txs stay mined. -func TestSetTxMinedStatus_BlockStillOnChain_NoRemark(t *testing.T) { - initPrometheusMetrics() +// TestRepairOvertakenTips_SharedTxWrittenAfterRepair: the other order for a tx B shares with +// the winning chain. The repair runs before the winning block's setTxMined has written the +// tx, so the tx still reads as only in B and is stamped; that setTxMined then clears it. +// Both run on the setMined worker, so they never overlap. +func TestRepairOvertakenTips_SharedTxWrittenAfterRepair(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _, blockB2ID := f.overtakeBPendingSharedWrite(t) - utxoStore, subtreeValidationClient, _, txStore, subtreeStore, deferFunc := setup(t) - defer deferFunc() + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.shared), "precondition: shared read as only in B, so it was stamped") - tSettings := test.CreateBaseTestSettings(t) - blockChainStore, err := blockchain_store.NewStore(ulogger.TestLogger{}, &url.URL{Scheme: "sqlitememory"}, tSettings) + f.mineSharedIn(t, blockB2ID) + + require.Zero(t, f.unminedSince(t, f.shared), "the winning block's write leaves shared mined") + require.NotZero(t, f.unminedSince(t, f.bOnly)) +} + +// TestRepairOvertakenTips_Slices: a block larger than one slice is repaired across several +// slices, each one a separate step of the setMined worker. +func TestRepairOvertakenTips_Slices(t *testing.T) { + f := newOvertakeFixture(t) + f.tSettings.BlockValidation.TipRepairSliceSize = 1 + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + steps, err := drainTipRepair(t, bv) require.NoError(t, err) - blockchainClient, err := blockchain.NewLocalClient(ulogger.TestLogger{}, tSettings, blockChainStore, nil, nil) + // Slices are over subtree nodes, the coinbase placeholder included: start the job, one + // step per node (the first also loads the subtree, the last finishes), find nothing left. + require.Equal(t, 5, steps) + + require.NotZero(t, f.unminedSince(t, f.bOnly)) + require.Zero(t, f.unminedSince(t, f.shared)) +} + +// TestRepairOvertakenTips_RepairsAgainAfterFlipBack: a block whose repair had nothing to +// re-mark (the winning chain mined all its txs) and that then comes back onto the chain must +// be repaired again when it leaves a second time, for a chain that mines none of them. +func TestRepairOvertakenTips_RepairsAgainAfterFlipBack(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + _, blockB2ID := f.overtakeBPendingSharedWrite(t) + f.mineSharedIn(t, blockB2ID) + f.mineIn(t, f.bOnly, blockB2ID) + + _, err := drainTipRepair(t, bv) require.NoError(t, err) + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: B' mined every tx of B, so nothing was re-marked") - coinbaseB, bOnlyTx, _, _ := createCoinbaseAndChildTx(t) + f.extendB(t) - _, _, err = utxoStore.SpendAndCreate(ctx, bOnlyTx, 1, utxostore.WithCreateOnly()) + _, err = drainTipRepair(t, bv) require.NoError(t, err) - subtree, err := subtreepkg.NewTreeByLeafCount(2) + // A third chain, longer than B -> D2 -> D3, that mines neither tx. + time.Sleep(10 * time.Millisecond) + + parent := f.tSettings.ChainCfgParams.GenesisHash + for height := uint32(1); height <= 4; height++ { + block := coinbaseOnlyBlock(t, parent, f.nBits, height, 0xe0+byte(height)) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, block, "test", blockchainoptions.WithSubtreesSet(true))) + parent = block.Hash() + } + + require.False(t, f.onChain(t, f.blockBID), "precondition: B is overtaken again") + + _, err = drainTipRepair(t, bv) require.NoError(t, err) - require.NoError(t, subtree.AddCoinbaseNode()) - require.NoError(t, subtree.AddNode(*bOnlyTx.TxIDChainHash(), uint64(bOnlyTx.Size()), 0)) //nolint:gosec + require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired again: no block on the new chain mines bOnly") +} - subtreeBytes, err := subtree.Serialize() +// TestRepairOvertakenTips_RetriesAfterFailure: a failed repair leaves the block tracked and +// unrepaired, so the next slice retries it. No later setTxMined can do it: B's mined_set is +// already written, and a retried setTxMined would see onLongestChain=false. +func TestRepairOvertakenTips_RetriesAfterFailure(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + f.store.failMarkFalse.Store(1 << 30) + + for i := 0; i < 2*tipRepairMaxAttempts; i++ { + _, err := drainTipRepair(t, bv) + require.Error(t, err) + } + + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: the failed repairs changed nothing") + _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.True(t, tracked, "a transient failure never gives the block up") + + f.store.failMarkFalse.Store(0) + + _, err := drainTipRepair(t, bv) require.NoError(t, err) - require.NoError(t, subtreeStore.Set(ctx, subtree.RootHash()[:], fileformat.FileTypeSubtree, subtreeBytes)) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the next attempt repairs the block") +} - replicated := subtree.Duplicate() - replicated.ReplaceRootNode(coinbaseB.TxIDChainHash(), 0, uint64(coinbaseB.Size())) //nolint:gosec +// blockGoneClient answers GetBlock with block-not-found. +type blockGoneClient struct { + blockchain.ClientI +} - nBits, err := model.NewNBitFromString("207fffff") +func (c *blockGoneClient) GetBlock(_ context.Context, blockHash *chainhash.Hash) (*model.Block, error) { + return nil, errors.NewBlockNotFoundError(blockHash.String()) +} + +// extraSubtreeClient returns blocks with an extra, never-stored subtree listed first. +type extraSubtreeClient struct { + blockchain.ClientI +} + +func (c *extraSubtreeClient) GetBlock(ctx context.Context, blockHash *chainhash.Hash) (*model.Block, error) { + block, err := c.ClientI.GetBlock(ctx, blockHash) + if err != nil { + return nil, err + } + + missing := chainhash.HashH([]byte("never stored")) + + return model.NewBlock(block.Header, block.CoinbaseTx, append([]*chainhash.Hash{&missing}, block.Subtrees...), + block.TransactionCount, block.SizeInBytes, block.Height, block.ID) +} + +// TestRepairOvertakenTips_SkipsMissingSubtree: a subtree that is gone must not stop the +// repair of the block's other subtrees, nor keep the block retrying. +func TestRepairOvertakenTips_SkipsMissingSubtree(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.blockchainClient = &extraSubtreeClient{ClientI: f.blockchainClient} + + _, err := drainTipRepair(t, bv) require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the readable subtree is repaired") + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "and the block is done, not retried") +} - headerB := mineBlockHeader(t, tSettings.ChainCfgParams.GenesisHash, replicated.RootHash(), nBits) - blockB, err := model.NewBlock(headerB, coinbaseB, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), uint64(coinbaseB.Size()+bOnlyTx.Size()), 1, 0) //nolint:gosec +// failingGetBlockClient fails GetBlock while fail is set: a blockchain service that is +// briefly unreachable. +type failingGetBlockClient struct { + blockchain.ClientI + fail atomic.Bool +} + +func (c *failingGetBlockClient) GetBlock(ctx context.Context, blockHash *chainhash.Hash) (*model.Block, error) { + if c.fail.Load() { + return nil, errors.NewServiceError("blockchain service unavailable") + } + + return c.ClientI.GetBlock(ctx, blockHash) +} + +// TestRepairOvertakenTips_TransientLoadFailureNeverGivesUp: failing to load the overtaken +// block because a service is briefly unavailable must not count toward giving the block up. +func TestRepairOvertakenTips_TransientLoadFailureNeverGivesUp(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + client := &failingGetBlockClient{ClientI: f.blockchainClient} + client.fail.Store(true) + bv.blockchainClient = client + + for i := 0; i < 2*tipRepairMaxAttempts; i++ { + _, err := drainTipRepair(t, bv) + require.Error(t, err) + } + + _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.True(t, tracked, "a transient failure never gives the block up") + + client.fail.Store(false) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly)) +} + +// TestRepairOvertakenTips_GivesUpAfterMaxAttempts: a block that can never be repaired (it is +// gone from the blockchain store) must not be retried, and logged, forever. +func TestRepairOvertakenTips_GivesUpAfterMaxAttempts(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.blockchainClient = &blockGoneClient{ClientI: f.blockchainClient} + + for i := 0; i < tipRepairMaxAttempts; i++ { + _, err := drainTipRepair(t, bv) + require.Error(t, err, "attempt %d", i+1) + } + + _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, tracked, "the block is dropped after the last attempt") + + _, err := drainTipRepair(t, bv) + require.NoError(t, err, "and no longer retried") +} + +// TestRepairOvertakenTips_FlipBackRemines: a repaired block whose chain becomes the longest +// again must have its txs marked mined again. +func TestRepairOvertakenTips_FlipBackRemines(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "precondition: repaired while B is off chain") + + f.extendB(t) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.Zero(t, f.unminedSince(t, f.bOnly), "B is on the longest chain again, so its txs are mined again") + require.Equal(t, []chainhash.Hash{*f.bOnly}, f.store.markedTrue, "only the tx the repair stamped is written back") +} + +// TestRepairOvertakenTips_SeedAfterRestart: the tracker is in memory. A block written mined +// before a restart and found off chain after it must still be repaired. The seed runs at the +// first slice after the restarted process marks a block mined, here the new tip C'. +func TestRepairOvertakenTips_SeedAfterRestart(t *testing.T) { + f := newOvertakeFixture(t) + + require.NoError(t, f.newBlockValidation(t).setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + restarted := f.newBlockValidation(t) + + _, err := drainTipRepair(t, restarted) require.NoError(t, err) + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: nothing tracked yet, so no repair has run") - require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test")) + require.NoError(t, restarted.setTxMinedStatus(f.ctx, blockC2.Hash())) - blockValidation := NewBlockValidation(ctx, ulogger.TestLogger{}, tSettings, blockchainClient, subtreeStore, txStore, utxoStore, nil, subtreeValidationClient) + _, err = drainTipRepair(t, restarted) + require.NoError(t, err) + + require.NotZero(t, f.unminedSince(t, f.bOnly), "a block overtaken across a restart must be repaired") + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") +} + +// TestRepairOvertakenTips_SkipsPrunedTxs: a tx of the overtaken block that is no longer in +// the store (pruned) must not stop the repair of the others. +func TestRepairOvertakenTips_SkipsPrunedTxs(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) - require.NoError(t, blockValidation.setTxMinedStatus(ctx, blockB.Hash())) + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) - bOnlyMeta, err := utxoStore.Get(ctx, bOnlyTx.TxIDChainHash(), fields.UnminedSince) + require.NoError(t, f.store.Store.Delete(f.ctx, f.shared)) + + _, err := drainTipRepair(t, bv) require.NoError(t, err) - require.Zero(t, bOnlyMeta.UnminedSince, "a tx mined in the tip must stay mined") + require.NotZero(t, f.unminedSince(t, f.bOnly), "the remaining txs are repaired") +} + +// TestRepairOvertakenTips_StillOnChainNoRemark is the control: when B stays the tip its txs +// stay mined. +func TestRepairOvertakenTips_StillOnChainNoRemark(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + + require.Zero(t, f.unminedSince(t, f.bOnly), "a tx mined in the tip must stay mined") +} + +// TestRepairOvertakenTips_RunsOnTheSetMinedWorker drives the production path end to end, with +// no test call into setTxMined or the repair: the running setMined worker marks B mined, +// the overtaking blocks' notifications ask it for repair slices, and it repairs B. +func TestRepairOvertakenTips_RunsOnTheSetMinedWorker(t *testing.T) { + f := newOvertakeFixture(t) + _ = f.newRunningBlockValidation(f.ctx) + + // The overtaking blocks' notifications must reach the worker, so wait for it to subscribe. + require.Eventually(t, func() bool { + subscribers, err := f.blockchainClient.GetSubscribers(f.ctx) + return err == nil && slices.Contains(subscribers, blockchain.SubscriberBlockValidation) + }, 10*time.Second, 10*time.Millisecond) + + // The worker's mined-not-set pass picks B up and writes it as on the longest chain. + require.Eventually(t, func() bool { + _, meta, err := f.blockchainClient.GetBlockHeader(f.ctx, f.blockB.Hash()) + return err == nil && meta.MinedSet + }, 10*time.Second, 10*time.Millisecond, "the setMined worker marks B mined") + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: B's txs are mined") + + _ = f.overtakeB(t) + + require.Eventually(t, func() bool { + m, err := f.store.Store.Get(f.ctx, f.bOnly, fields.UnminedSince) + return err == nil && m.UnminedSince != 0 + }, 10*time.Second, 20*time.Millisecond, "the setMined worker repairs the overtaken block") + + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") +} + +// grpcLikeBestHeaderClient returns GetBestBlockHeader metas without the block id, as the +// gRPC blockchain client does. +type grpcLikeBestHeaderClient struct { + blockchain.ClientI +} + +func (c *grpcLikeBestHeaderClient) GetBestBlockHeader(ctx context.Context) (*model.BlockHeader, *model.BlockHeaderMeta, error) { + header, meta, err := c.ClientI.GetBestBlockHeader(ctx) + if meta != nil { + stripped := *meta + stripped.ID = 0 + meta = &stripped + } + + return header, meta, err +} + +// TestMainChainIDs_ExtensionWithoutBestHeaderID: when the best block only extends the one +// seen last time, its id must still be added to the main-chain set even though +// GetBestBlockHeader over gRPC carries no id. Adding id 0 instead made the new tip look off +// the chain, and the repair stamped its txs unmined. +func TestMainChainIDs_ExtensionWithoutBestHeaderID(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + client := &grpcLikeBestHeaderClient{ClientI: f.blockchainClient} + + _, _, err := bv.mainChainIDsLocked(f.ctx, client) + require.NoError(t, err) + + time.Sleep(10 * time.Millisecond) + + child := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xa2) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, child, "test", blockchainoptions.WithSubtreesSet(true))) + + _, childMeta, err := f.blockchainClient.GetBlockHeader(f.ctx, child.Hash()) + require.NoError(t, err) + + onMainChain, _, err := bv.mainChainIDsLocked(f.ctx, client) + require.NoError(t, err) + require.True(t, onMainChain[childMeta.ID], "the new tip is on the main chain") + require.True(t, onMainChain[f.blockBID]) +} + +func TestTipRepairExpired(t *testing.T) { + // GetBlockHeaderIDs(best, tipRepairWindow) covers best down to best-tipRepairWindow+1. + require.False(t, tipRepairExpired(1001, 2000), "inside the main chain ID window") + require.True(t, tipRepairExpired(1000, 2000), "outside it: treating it as off chain would read every tx") + require.False(t, tipRepairExpired(5, 10), "young chain") +} + +func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { + tracker := newMinedOnChainTracker() + hash := chainhash.HashH([]byte("b")) + + tracker.track(hash, 1, 1, false) + require.True(t, tracker.markWritten(hash)) + + gen := tracker.snapshot()[hash].gen + tracker.update(hash, gen, func(e *minedOnChainBlock) { + e.stamped = true + e.done = true + }) + + // setTxMined writes the block as on the longest chain again (its mined_set was cleared). + tracker.track(hash, 1, 1, false) + + b := tracker.snapshot()[hash] + require.NotEqual(t, gen, b.gen, "a new write is a new generation; a repair in progress for the old one is abandoned") + require.False(t, b.done, "the new write clears the txs again, so the block needs repairing again") + require.False(t, b.stamped) + require.False(t, b.written, "and must not be repaired until the new write completes") + + tracker.update(hash, gen, func(e *minedOnChainBlock) { e.done = true }) + require.False(t, tracker.snapshot()[hash].done, "an update for the old generation is ignored") } diff --git a/settings.conf b/settings.conf index 5263de3fe1..d5d79c737b 100644 --- a/settings.conf +++ b/settings.conf @@ -644,6 +644,11 @@ blockvalidation_recentBlockIDsLimit = 50000 blockvalidation_recentBlockIDsLimit.docker.m = 10000 blockvalidation_recentBlockIDsLimit.dev = 1000 +# Transactions re-marked per step of the overtaken-tip repair, which runs on the setMined +# worker between blocks (issue 2051). Bounds how long one step delays the next block's +# mined status. +blockvalidation_tipRepairSliceSize = 250000 + # Skip the per-transaction UTXO lock/unlock during quick-validation catchup (default: false). # Quick validation runs only for blocks at or below the highest checkpoint. Normally each tx's # UTXOs are created locked and unlocked again after the block is committed — two Aerospike record diff --git a/settings/blockvalidation_settings.go b/settings/blockvalidation_settings.go index 9e77d7fee1..b1fc0fdac2 100644 --- a/settings/blockvalidation_settings.go +++ b/settings/blockvalidation_settings.go @@ -73,6 +73,7 @@ type BlockValidationSettings struct { MaxBlocksBehindBlockAssembly int `key:"blockvalidation_maxBlocksBehindBlockAssembly" desc:"Maximum blocks behind Block Assembly" default:"20" category:"BlockValidation" usage:"Sync tolerance" type:"int" longdesc:"### Purpose\nSets the maximum blocks Block Validation can lag behind Block Assembly.\n\n### How It Works\n- Block Assembly may produce blocks faster than Block Validation can process\n- When this threshold is exceeded, catchup mode or waiting is triggered\n- Prevents unbounded queue growth between services\n\n### Recommendations\n- **20** (default) - Reasonable buffer for processing delays\n- Increase for systems with variable validation times\n- Decrease to maintain tighter synchronization"` PeriodicProcessingInterval time.Duration `key:"blockvalidation_periodic_processing_interval" desc:"Interval for periodic processing" default:"1m" category:"BlockValidation" usage:"Background task frequency" type:"duration" longdesc:"### Purpose\nSets the interval for periodic background processing tasks.\n\n### Tasks Performed\n- Checking mined status of pending blocks\n- Cache cleanup and maintenance\n- Resource cleanup and garbage collection\n\n### Recommendations\n- **1m** (default) - Regular maintenance without excessive overhead\n- Decrease for faster status updates (more CPU usage)\n- Increase for reduced background activity"` RecentBlockIDsLimit uint64 `key:"blockvalidation_recentBlockIDsLimit" desc:"Maximum recent block IDs to load" default:"50000" category:"BlockValidation" usage:"Memory limit for recent block IDs" type:"uint64" longdesc:"### Purpose\nLimits the number of recent block IDs loaded into memory for double-spend checking.\n\n### How It Works\nRecent block IDs are kept in memory to support fast double-spend detection. This setting prevents unbounded memory growth.\n\n### Recommendations\n- **50000** (default) - Covers typical scenarios\n- Increase for deeper history requirements\n- Decrease for memory-constrained environments"` + TipRepairSliceSize int `key:"blockvalidation_tipRepairSliceSize" desc:"Transactions re-marked per overtaken-tip repair slice" default:"250000" category:"BlockValidation" usage:"Bounds how long one repair slice holds the setMined worker" type:"int" longdesc:"### Purpose\nBounds the work of one slice of the overtaken-tip repair.\n\n### How It Works\nWhen a block whose transactions were marked mined is overtaken before block assembly moved onto it, block validation re-marks the transactions that are not mined on the new chain. It does this on the setMined worker, between blocks, a slice of transactions at a time, so the next block's mined status is delayed by at most one slice.\n\n### Recommendations\n- **250000** (default)\n- Decrease if setTxMined latency matters more than repair speed"` // Catchup configuration CatchupChBufferSize int `key:"blockvalidation_catchupCh_buffer_size" desc:"Buffer size for catchup channel" default:"100" category:"BlockValidation" usage:"Queue for catchup blocks" type:"int" longdesc:"### Purpose\nSets the buffer size for the channel receiving catchup block notifications.\n\n### How It Works\nDuring catchup mode, blocks are queued for processing. This buffer prevents blocking when multiple blocks arrive quickly.\n\n### Recommendations\n- **100** (default) - Handles typical catchup scenarios\n- Increase if experiencing backpressure during large syncs"` UseCatchupWhenBehind bool `key:"blockvalidation_useCatchupWhenBehind" desc:"Enable automatic catchup mode" default:"false" category:"BlockValidation" usage:"Auto-switch to catchup" type:"bool" longdesc:"### Purpose\nAutomatically switches to optimized catchup mode when the node falls behind the network.\n\n### How It Works\n- Monitors chain height relative to known network tip\n- When behind by threshold, switches to catchup mode\n- Catchup uses batch fetching and quick validation for faster sync\n- Returns to normal mode once caught up\n\n### Values\n- **false** (default) - Always use normal validation\n- **true** - Enable automatic catchup mode switching\n\n### Recommendations\n- Enable for nodes that may experience periods of being offline\n- Keep disabled for always-connected mining nodes"` diff --git a/settings/settings.go b/settings/settings.go index f5d271a0b0..ed6c361654 100644 --- a/settings/settings.go +++ b/settings/settings.go @@ -431,6 +431,7 @@ func NewSettings(alternativeContext ...string) *Settings { MaxBlocksBehindBlockAssembly: getInt("blockvalidation_maxBlocksBehindBlockAssembly", 20, alternativeContext...), PeriodicProcessingInterval: getDuration("blockvalidation_periodic_processing_interval", 1*time.Minute, alternativeContext...), RecentBlockIDsLimit: getUint64("blockvalidation_recentBlockIDsLimit", 50000, alternativeContext...), + TipRepairSliceSize: getInt("blockvalidation_tipRepairSliceSize", 250000, alternativeContext...), SubtreeMmapDir: getString("blockvalidation_subtreeMmapDir", "", alternativeContext...), // Catchup configuration CatchupChBufferSize: getInt("blockvalidation_catchupCh_buffer_size", 100, alternativeContext...), diff --git a/stores/blockchain/mock.go b/stores/blockchain/mock.go index 5703df5dfc..df80d7a9dd 100644 --- a/stores/blockchain/mock.go +++ b/stores/blockchain/mock.go @@ -13,8 +13,10 @@ package blockchain import ( + "cmp" "context" "net/http" + "slices" "sync" "time" @@ -327,8 +329,46 @@ func (m *MockStore) AssignBlockID(_ context.Context, blockHash *chainhash.Hash) return id, nil } +// GetLastNBlocks returns up to n blocks, highest first. Without includeOrphans only the best +// block's chain is returned; fromHeight, when non-zero, caps the height. func (m *MockStore) GetLastNBlocks(ctx context.Context, n int64, includeOrphans bool, fromHeight uint32) ([]*model.BlockInfo, error) { - panic(implementMe) + m.mu.RLock() + defer m.mu.RUnlock() + + mainChain := make(map[chainhash.Hash]bool) + if m.BestBlock != nil { + for block := m.BestBlock; block != nil; block = m.Blocks[*block.Header.HashPrevBlock] { + mainChain[*block.Hash()] = true + } + } + + blocks := make([]*model.Block, 0, len(m.Blocks)) + for hash, block := range m.Blocks { + if (!includeOrphans && !mainChain[hash]) || (fromHeight > 0 && block.Height > fromHeight) { + continue + } + + blocks = append(blocks, block) + } + + slices.SortFunc(blocks, func(a, b *model.Block) int { return cmp.Compare(b.Height, a.Height) }) + + infos := make([]*model.BlockInfo, 0, min(int64(len(blocks)), n)) + for _, block := range blocks { + if int64(len(infos)) >= n { + break + } + + infos = append(infos, &model.BlockInfo{ + Height: block.Height, + Orphaned: !mainChain[*block.Hash()], + BlockHeader: block.Header.Bytes(), + TransactionCount: block.TransactionCount, + Size: block.SizeInBytes, + }) + } + + return infos, nil } func (m *MockStore) GetLastNInvalidBlocks(ctx context.Context, n int64) ([]*model.BlockInfo, error) { diff --git a/stores/txmetacache/txmetacache.go b/stores/txmetacache/txmetacache.go index 6cd156e3d9..31b33e4fa2 100644 --- a/stores/txmetacache/txmetacache.go +++ b/stores/txmetacache/txmetacache.go @@ -1503,8 +1503,8 @@ func (t *TxMetaCache) PendingConflictIntents(ctx context.Context) ([]utxo.Confli // // Returns: // - Error if the operation fails -func (t *TxMetaCache) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { - return t.utxoStore.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) +func (t *TxMetaCache) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { + return t.utxoStore.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) } // SetBlockHeight updates the current block height in the underlying store. diff --git a/stores/utxo/Interface.go b/stores/utxo/Interface.go index 0624bb358f..3be85bec39 100644 --- a/stores/utxo/Interface.go +++ b/stores/utxo/Interface.go @@ -464,6 +464,35 @@ func WithSkipUTXOHashCheck(b bool) CreateOption { return optSkipUTXOHashCheckFalse } +// MarkLongestChainOptions configures MarkTransactionsOnLongestChain. +type MarkLongestChainOptions struct { + // SkipMissing makes a record missing from the store a logged skip instead of fatal for + // onLongestChain=false. + SkipMissing bool +} + +// MarkLongestChainOption is a functional option for MarkTransactionsOnLongestChain. +type MarkLongestChainOption func(*MarkLongestChainOptions) + +// WithSkipMissing is for a caller whose records may legitimately have been pruned since it +// read them, such as block validation's overtaken-tip repair, which works on old blocks +// while the pruner runs. Everyone else keeps a missing record fatal: it means corruption. +func WithSkipMissing() MarkLongestChainOption { + return func(o *MarkLongestChainOptions) { + o.SkipMissing = true + } +} + +// NewMarkLongestChainOptions applies opts to a zero MarkLongestChainOptions. +func NewMarkLongestChainOptions(opts ...MarkLongestChainOption) *MarkLongestChainOptions { + o := &MarkLongestChainOptions{} + for _, opt := range opts { + opt(o) + } + + return o +} + // WithCreateOnly makes SpendAndCreate skip the spend phase: only the // transaction's outputs and metadata are stored (coinbase, seeding, and batch // flows whose inputs are spent elsewhere). @@ -836,8 +865,9 @@ type Store interface { // MarkTransactionsOnLongestChain marks transactions as being on the longest chain or not. // When onLongestChain is true, the unminedSince field is unset (transaction is mined). // When onLongestChain is false, an existing unminedSince is kept; only an absent value is - // stamped, with BackdatedUnminedSince (issue 1768). - MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error + // stamped, with BackdatedUnminedSince (issue 1768). A record missing from the store is + // fatal for onLongestChain=false unless WithSkipMissing is given. + MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...MarkLongestChainOption) error // internal state functions diff --git a/stores/utxo/aerospike/longest_chain.go b/stores/utxo/aerospike/longest_chain.go index 025a45cf87..0b80af56d0 100644 --- a/stores/utxo/aerospike/longest_chain.go +++ b/stores/utxo/aerospike/longest_chain.go @@ -64,7 +64,7 @@ import ( // That is deliberate: the reset reloads the unmined transactions and re-marks the ones the master // restore (applyLongestChainChildDAH) left in a mined state with unmined_since set, so nothing is lost, // at the price of one transient page failure costing an incremental reorg. Do not swallow this error. -func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { if len(txHashes) == 0 { return nil } @@ -265,10 +265,15 @@ func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []c attempted, succeeded, finalErrorCount, finalMissingCount, atomic.LoadInt64(&childDAHFailed), onLongestChain) if finalMissingCount > 0 { - if !onLongestChain { + switch skipMissing := utxo.NewMarkLongestChainOptions(opts...).SkipMissing; { + case !onLongestChain && !skipMissing: s.logger.Fatalf("CRITICAL: %d missing transactions during MarkTransactionsOnLongestChain(false) - data integrity compromised, first: %v", finalMissingCount, firstMissing) - } else { + case skipMissing: + // Expected: the caller's records may have been pruned since it read them. + s.logger.Warnf("[MarkTransactionsOnLongestChain] skipped %d transactions no longer in the store, first: %v", + finalMissingCount, firstMissing) + default: s.logger.Errorf("[MarkTransactionsOnLongestChain] skipped %d transactions the store does not hold, first: %v", finalMissingCount, firstMissing) } diff --git a/stores/utxo/factory/utxo_test.go b/stores/utxo/factory/utxo_test.go index 0b5d9c518b..4e573d68ad 100644 --- a/stores/utxo/factory/utxo_test.go +++ b/stores/utxo/factory/utxo_test.go @@ -205,7 +205,7 @@ func (m *MockUTXOStore) PendingConflictIntents(ctx context.Context) ([]utxo.Conf return nil, nil } -func (m *MockUTXOStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (m *MockUTXOStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { args := m.Called(ctx, txHashes, onLongestChain) return args.Error(0) } diff --git a/stores/utxo/logger/logger.go b/stores/utxo/logger/logger.go index eb3b903e8e..df3ca821bd 100644 --- a/stores/utxo/logger/logger.go +++ b/stores/utxo/logger/logger.go @@ -393,8 +393,8 @@ func (s *Store) PendingConflictIntents(ctx context.Context) ([]utxo.ConflictInte return intents, err } -func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { - err := s.store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain) +func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { + err := s.store.MarkTransactionsOnLongestChain(ctx, txHashes, onLongestChain, opts...) s.logger.Debugf("[UTXOStore][logger][MarkTransactionsOnLongestChain] txHashes %v onLongestChain %v err %v : %s", txHashes, onLongestChain, err, caller()) return err diff --git a/stores/utxo/logger/logger_test.go b/stores/utxo/logger/logger_test.go index 55a91f7622..b925bb5577 100644 --- a/stores/utxo/logger/logger_test.go +++ b/stores/utxo/logger/logger_test.go @@ -266,7 +266,7 @@ func (m *MockStore) PendingConflictIntents(ctx context.Context) ([]utxo.Conflict return args.Get(0).([]utxo.ConflictIntent), args.Error(1) } -func (m *MockStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (m *MockStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { args := m.Called(ctx, txHashes, onLongestChain) return args.Error(0) } diff --git a/stores/utxo/mock.go b/stores/utxo/mock.go index 8dd31b82f0..fe765641b9 100644 --- a/stores/utxo/mock.go +++ b/stores/utxo/mock.go @@ -370,7 +370,7 @@ func (m *MockUtxostore) PendingConflictIntents(ctx context.Context) ([]ConflictI // MarkTransactionsOnLongestChain mocks the marking of transactions as being on the longest chain. // Returns the configured mock response for longest chain marking operations. -func (m *MockUtxostore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (m *MockUtxostore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...MarkLongestChainOption) error { args := m.Called(ctx, txHashes, onLongestChain) return args.Error(0) } diff --git a/stores/utxo/nullstore/nullstore.go b/stores/utxo/nullstore/nullstore.go index e89ba2f1aa..6fa62831e5 100644 --- a/stores/utxo/nullstore/nullstore.go +++ b/stores/utxo/nullstore/nullstore.go @@ -326,7 +326,7 @@ func (m *NullStore) PendingConflictIntents(ctx context.Context) ([]utxo.Conflict return nil, nil } -func (m *NullStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (m *NullStore) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { return nil } diff --git a/stores/utxo/sql/longest_chain_skip_missing_test.go b/stores/utxo/sql/longest_chain_skip_missing_test.go new file mode 100644 index 0000000000..e5df3e3910 --- /dev/null +++ b/stores/utxo/sql/longest_chain_skip_missing_test.go @@ -0,0 +1,89 @@ +package sql + +import ( + "context" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/teranode/stores/utxo" + "github.com/bsv-blockchain/teranode/stores/utxo/fields" + "github.com/bsv-blockchain/teranode/ulogger" + "github.com/bsv-blockchain/teranode/util/test" + "github.com/stretchr/testify/require" +) + +// fatalRecordingLogger records Fatalf instead of exiting. +type fatalRecordingLogger struct { + ulogger.TestLogger + fatal atomic.Bool +} + +func (l *fatalRecordingLogger) Fatalf(format string, args ...interface{}) { + l.fatal.Store(true) +} + +// TestMarkTransactionsOnLongestChain_SkipMissing: a not-on-longest-chain mark over records +// that may have been pruned since they were read (the overtaken-tip repair races the +// pruner) must skip the missing ones instead of treating them as corruption, and still +// stamp the rest. Without the option a missing record stays fatal. +func TestMarkTransactionsOnLongestChain_SkipMissing(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + initPrometheusMetrics() + + tSettings := test.CreateBaseTestSettings(t) + tSettings.UtxoStore.DBTimeout = 30 * time.Second + tSettings.BatcherDrainMode = true + + utxoStoreURL, err := url.Parse("sqlitememory:///test") + require.NoError(t, err) + + logger := &fatalRecordingLogger{} + store, err := New(ctx, logger, tSettings, utxoStoreURL) + require.NoError(t, err) + + _, tx := setup(ctx, t) + _, err = store.Create(ctx, tx, 100, utxo.WithMinedBlockInfo(utxo.MinedBlockInfo{BlockID: 7, BlockHeight: 100, OnLongestChain: true})) + require.NoError(t, err) + require.NoError(t, store.SetBlockHeight(200)) + + missing, err := chainhash.NewHashFromStr("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef") + require.NoError(t, err) + + require.NoError(t, store.MarkTransactionsOnLongestChain(ctx, []chainhash.Hash{*tx.TxIDChainHash(), *missing}, false, utxo.WithSkipMissing())) + require.False(t, logger.fatal.Load(), "a missing record is not fatal with WithSkipMissing") + + m, err := store.Get(ctx, tx.TxIDChainHash(), fields.UnminedSince) + require.NoError(t, err) + require.NotZero(t, m.UnminedSince, "the record that exists is still stamped") +} + +// TestMarkTransactionsOnLongestChain_MissingStaysFatal: without the option a missing record +// on a not-on-longest-chain mark still means corruption. +func TestMarkTransactionsOnLongestChain_MissingStaysFatal(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + initPrometheusMetrics() + + tSettings := test.CreateBaseTestSettings(t) + tSettings.UtxoStore.DBTimeout = 30 * time.Second + tSettings.BatcherDrainMode = true + + utxoStoreURL, err := url.Parse("sqlitememory:///test") + require.NoError(t, err) + + logger := &fatalRecordingLogger{} + store, err := New(ctx, logger, tSettings, utxoStoreURL) + require.NoError(t, err) + + missing, err := chainhash.NewHashFromStr("1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcdef") + require.NoError(t, err) + + _ = store.MarkTransactionsOnLongestChain(ctx, []chainhash.Hash{*missing}, false) + require.True(t, logger.fatal.Load()) +} diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index fbded9aa4a..01c346bf64 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -5232,7 +5232,7 @@ func (s *Store) existingHashes(ctx context.Context, q string, args []interface{} return found, rows.Err() } -func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error { +func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...utxo.MarkLongestChainOption) error { if len(txHashes) == 0 { return nil } @@ -5380,10 +5380,15 @@ func (s *Store) MarkTransactionsOnLongestChain(ctx context.Context, txHashes []c attempted, totalUpdated, errorCount, missingCount, onLongestChain) if missingCount > 0 { - if !onLongestChain { + switch skipMissing := utxo.NewMarkLongestChainOptions(opts...).SkipMissing; { + case !onLongestChain && !skipMissing: s.logger.Fatalf("CRITICAL: %d MISSING transactions during MarkTransactionsOnLongestChain(false) - data integrity compromised, first: %v", missingCount, firstMissing) - } else { + case skipMissing: + // Expected: the caller's records may have been pruned since it read them. + s.logger.Warnf("[MarkTransactionsOnLongestChain] skipped %d transactions no longer in the store, first: %v", + missingCount, firstMissing) + default: s.logger.Errorf("[MarkTransactionsOnLongestChain] skipped %d transactions the store does not hold, first: %v", missingCount, firstMissing) } From 116eed14a0e54d551107856b2c1c2e6b704854c6 Mon Sep 17 00:00:00 2001 From: Siggi Date: Mon, 26 Oct 2026 19:56:14 +0100 Subject: [PATCH 03/33] fix(test): data race in the memory blob store and three order- or timing-dependent tests Surfaced by -race runs and Aerospike runs on this branch; none is caused by it. - stores/blob/memory: currentBlockHeight was written by SetCurrentBlockHeight and read by the TTL cleaner goroutine without synchronisation. It is now an atomic.Uint32. - TestIntegrationRetryWithMultipleFailures sent all three peer announcements at once. When peer3 stored the block first, the failing peers never fetched and their attempt counters stayed at 0. Announcements now go out one at a time, each after the previous peer has been tried. - The commitBlock harness relied on another test having initialised the prometheus metrics; run on its own, commitBlock dereferenced a nil histogram. - TestUnminedSinceReorgScenarios generated its 1005 setup blocks in one RPC call, which outlasts the 30s RPC timeout on Aerospike. It now generates them in chunks. Refs #2051 --- services/blockvalidation/commit_block_test.go | 1 + services/blockvalidation/integration_retry_test.go | 10 +++++++++- stores/blob/memory/memory.go | 11 ++++++----- test/e2e/daemon/wip/unmined_since_reorg_bug_test.go | 7 +++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/services/blockvalidation/commit_block_test.go b/services/blockvalidation/commit_block_test.go index a5b5909569..1995f50426 100644 --- a/services/blockvalidation/commit_block_test.go +++ b/services/blockvalidation/commit_block_test.go @@ -32,6 +32,7 @@ import ( // The block has no subtrees so unlockSubtreeTransactionsIfNeeded is a no-op. func newCommitBlockHarness(t *testing.T) (*BlockValidation, *model.Block, context.Context) { t.Helper() + initPrometheusMetrics() ctx := context.Background() logger := ulogger.TestLogger{} diff --git a/services/blockvalidation/integration_retry_test.go b/services/blockvalidation/integration_retry_test.go index b9355eddf1..007bec925c 100644 --- a/services/blockvalidation/integration_retry_test.go +++ b/services/blockvalidation/integration_retry_test.go @@ -138,7 +138,11 @@ func TestIntegrationRetryWithMultipleFailures(t *testing.T) { {"http://peer3", "peer3"}, } - // Send announcements from all peers + attempts := map[string]*int32{"peer1": &peer1Attempts, "peer2": &peer2Attempts, "peer3": &peer3Attempts} + + // Send announcements from all peers, one at a time: once peer3 has stored the + // block, a later announcement finds it already present and never fetches, so the + // failing peers must each be tried before the next announcement goes out. for _, peer := range peers { kafkaMsg := &kafkamessage.KafkaBlockTopicMessage{ Hash: targetHash.String(), @@ -158,6 +162,10 @@ func TestIntegrationRetryWithMultipleFailures(t *testing.T) { // We expect this to succeed eventually assert.NoError(t, handler(kafkaMessage)) }() + + peerAttempts := attempts[peer.peerID] + require.Eventually(t, func() bool { return atomic.LoadInt32(peerAttempts) >= 1 }, 10*time.Second, 10*time.Millisecond, + "%s must be tried before the next announcement", peer.peerID) } // Wait for processing with retries diff --git a/stores/blob/memory/memory.go b/stores/blob/memory/memory.go index 7bffd4d8e5..fa2fb0cef9 100644 --- a/stores/blob/memory/memory.go +++ b/stores/blob/memory/memory.go @@ -22,6 +22,7 @@ import ( "io" "net/http" "sync" + "sync/atomic" "time" "github.com/bsv-blockchain/teranode/errors" @@ -45,7 +46,7 @@ type Memory struct { options *options.Options Counters map[string]int countersMu sync.Mutex - currentBlockHeight uint32 + currentBlockHeight atomic.Uint32 stopCleaner context.CancelFunc } @@ -85,7 +86,7 @@ func New(opts ...options.StoreOption) *Memory { // Parameters: // - blockHeight: The current blockchain height func (m *Memory) SetBlockHeight(blockHeight uint32) { - m.currentBlockHeight = blockHeight + m.currentBlockHeight.Store(blockHeight) } // ttlCleaner runs a periodic cleanup process to remove expired blobs. @@ -101,7 +102,7 @@ func (m *Memory) ttlCleaner(ctx context.Context, interval time.Duration) { case <-ctx.Done(): return case <-time.After(interval): - cleanExpiredFiles(m, m.currentBlockHeight) + cleanExpiredFiles(m, m.currentBlockHeight.Load()) } } } @@ -222,7 +223,7 @@ func (m *Memory) Set(ctx context.Context, key []byte, fileType fileformat.FileTy dah := merged.DAH if dah == 0 && merged.BlockHeightRetention > 0 { - dah = m.currentBlockHeight + merged.BlockHeightRetention + dah = m.currentBlockHeight.Load() + merged.BlockHeightRetention } var data []byte @@ -344,5 +345,5 @@ func (m *Memory) ListKeys() [][]byte { } func (m *Memory) SetCurrentBlockHeight(height uint32) { - m.currentBlockHeight = height + m.currentBlockHeight.Store(height) } diff --git a/test/e2e/daemon/wip/unmined_since_reorg_bug_test.go b/test/e2e/daemon/wip/unmined_since_reorg_bug_test.go index e08f5fe8d5..717f9abb10 100644 --- a/test/e2e/daemon/wip/unmined_since_reorg_bug_test.go +++ b/test/e2e/daemon/wip/unmined_since_reorg_bug_test.go @@ -117,8 +117,11 @@ func runReorgScenario(t *testing.T, storeType, scenario string, expectedBefore, // Step 1: Mine to height > 1000 (required for large reorg path) t.Log("Step 1: Mining to height > 1000...") var forkPointHeight uint32 - _, err := td.CallRPC(ctx, "generate", []any{1005}) - require.NoError(t, err) + // In chunks: a single generate of 1005 blocks outlasts the 30s RPC timeout on Aerospike. + for range 5 { + _, err := td.CallRPC(ctx, "generate", []any{201}) + require.NoError(t, err) + } currentHeight, _, err := td.BlockchainClient.GetBestHeightAndTime(ctx) require.NoError(t, err) From 401a1c7a985977f924c2023b474c845a47a3e775 Mon Sep 17 00:00:00 2001 From: Siggi Date: Tue, 27 Oct 2026 16:10:01 +0100 Subject: [PATCH 04/33] test(settings): count the tipRepairSliceSize row in the blockvalidation settings doc --- settings/settings_doc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/settings_doc_test.go b/settings/settings_doc_test.go index b1b7278484..a458242294 100644 --- a/settings/settings_doc_test.go +++ b/settings/settings_doc_test.go @@ -901,7 +901,7 @@ var checkedRowsPerDoc = map[string]int{ "blockassembly_settings.md": 44, "blockchain_settings.md": 11, "blockpersister_settings.md": 9, - "blockvalidation_settings.md": 70, + "blockvalidation_settings.md": 71, "coinbase_settings.md": 21, "faucet_settings.md": 1, "legacy_settings.md": 28, From aa08cf60083308eb172f4dbca3dd780307c3f42d Mon Sep 17 00:00:00 2001 From: Siggi Date: Tue, 27 Oct 2026 17:01:59 +0100 Subject: [PATCH 05/33] fix(blockvalidation): repair a fork block whose setTxMined failed part way before a restart A setTxMined that cleared some of a block's txs and then failed leaves the block's mined_set unwritten. After a restart the seed skipped such blocks, and their retry, with onLongestChain=false once overtaken, leaves unmined_since alone, so the cleared txs kept looking mined while off the chain. The seed now tracks a nearby fork block whose mined_set is not written as unwritten, so its retry makes it repairable. Also covers the repair's error paths with tests, documents the remaining restart limitation and the multi-replica race, lists MarkTransactionsOnLongestChain for block validation in the UTXO store write-access docs, and adds the CHANGELOG entry. --- CHANGELOG.md | 11 + docs/topics/stores/utxo.md | 4 +- services/blockvalidation/BlockValidation.go | 2 +- .../blockvalidation/overtaken_tip_repair.go | 49 +++- .../settxmined_overtaken_test.go | 239 +++++++++++++++++- 5 files changed, 282 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 23a86e85aa..506020f7a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -106,6 +106,17 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea ### Breaking — behaviour changes on upgrade +- **A reorg no longer re-marks block assembly's whole mempool.** Block assembly marks + only the transactions it moved back and that are not mined on the new chain. Block + validation now repairs a block it marked mined that a competing chain later overtakes, + on its setMined worker, in slices of `blockvalidation_tipRepairSliceSize` transactions + (new setting, default 250000). Each slice can delay the next block's mined status by + one slice. The repair assumes a single block validation instance writes mined status + for the chain, and its state is in memory: fork blocks more than 10 below the tip at a + restart are not repaired. A deployment that runs more than one block validation + replica loses no data, but transactions can stay flagged unmined until block + assembly's next unmined load. No action needed for a single-instance deployment. (#2065) + - **A block's tx is created as conflicting only when it is a double spend and nothing else.** Block validation used to create a tx as conflicting as soon as one of its inputs was already spent by another tx, dropping every other failed input. A block holding a diff --git a/docs/topics/stores/utxo.md b/docs/topics/stores/utxo.md index 26a6e112d2..d6609612d0 100644 --- a/docs/topics/stores/utxo.md +++ b/docs/topics/stores/utxo.md @@ -632,8 +632,8 @@ The current writers, re-derived directly from the code (not from the settings/de | Service | File(s) | Writes | Why | |---|---|---|---| | **Validator** | `services/validator/Validator.go` | `SpendAndCreate` (creates the tx's outputs, spends its inputs), `SetLocked` | Core transaction validation: on accepting a new transaction it atomically creates its UTXOs and spends the ones it consumes. `SetLocked` completes a two-phase commit on the tx's own hash: when the tx is queued for block assembly its outputs are created locked (`WithLocked(true)`), and unlocked only once it has been added successfully (`twoPhaseCommitTransaction`) — a crash in between leaves them unspendable rather than spendable. | -| **Block Validation** | `services/blockvalidation/BlockValidation.go`, `services/blockvalidation/quick_validate.go` | `SpendAndCreate` (`WithCreateOnly`/`WithSpendOnly`, quick-validate replay), `SetMinedMulti` (steady state, via `model.UpdateTxMinedStatus` in `setTxMined`), `utxo.SetMinedMultiChunked` (→ `SetMinedMulti`, quick-validate replay), `SetLocked` | Validates incoming blocks: creates/spends UTXOs for transactions not already known, marks the block's transactions as mined (records `BlockID`/`SubtreeIdx`), then — unless `quickValidateSkipsUtxoLock` applies to this block — unlocks the block's own subtree transactions at the end of quick-validate (`unlockSubtreeTransactions`). | -| **Block Assembly** | `services/blockassembly/BlockAssembler.go`, `services/blockassembly/subtreeprocessor/SubtreeProcessor.go` | `SpendAndCreate` (`WithCreateOnly` — creates the coinbase UTXOs, `processCoinbaseUtxos`), `Delete` (coinbase and its child spends on reorg, `removeCoinbaseUtxos` — Block Assembly is the only service that calls `Delete` directly), `MarkTransactionsOnLongestChain`, `SetLocked` (unlocks unmined transactions reloaded at startup; unlocks conflict-locked parents when healing a stale conflict intent), `utxo.ProcessConflicting` / `utxo.ReverseProcessConflicting` (→ `SetConflicting`, `Unspend`, `SpendAndCreate`, `BeginConflictIntent`/`CompleteConflictIntent`) | Owns chain-tip bookkeeping during block building and reorgs: creates the coinbase UTXO and deletes it (with its child spends) on reorg, flags which transactions sit on the currently-longest chain, unlocks unmined transactions reloaded at startup and conflict-locked parents when healing a stale intent, and drives the conflicting-transaction state machine (marking/unmarking double-spends and reversing that on reorg). | +| **Block Validation** | `services/blockvalidation/BlockValidation.go`, `services/blockvalidation/quick_validate.go`, `services/blockvalidation/overtaken_tip_repair.go` | `SpendAndCreate` (`WithCreateOnly`/`WithSpendOnly`, quick-validate replay), `SetMinedMulti` (steady state, via `model.UpdateTxMinedStatus` in `setTxMined`), `utxo.SetMinedMultiChunked` (→ `SetMinedMulti`, quick-validate replay), `SetLocked`, `MarkTransactionsOnLongestChain` (overtaken-tip repair, `overtaken_tip_repair.go`) | Validates incoming blocks: creates/spends UTXOs for transactions not already known, marks the block's transactions as mined (records `BlockID`/`SubtreeIdx`), re-marks the transactions of a block it marked mined that has since left the longest chain, then — unless `quickValidateSkipsUtxoLock` applies to this block — unlocks the block's own subtree transactions at the end of quick-validate (`unlockSubtreeTransactions`). | +| **Block Assembly** | `services/blockassembly/BlockAssembler.go`, `services/blockassembly/subtreeprocessor/SubtreeProcessor.go` | `SpendAndCreate` (`WithCreateOnly` — creates the coinbase UTXOs, `processCoinbaseUtxos`), `Delete` (coinbase and its child spends on reorg, `removeCoinbaseUtxos` — Block Assembly is the only service that calls `Delete` directly), `MarkTransactionsOnLongestChain`, `SetLocked` (unlocks unmined transactions reloaded at startup; unlocks conflict-locked parents when healing a stale conflict intent), `utxo.ProcessConflicting` / `utxo.ReverseProcessConflicting` (→ `SetConflicting`, `Unspend`, `SpendAndCreate`, `BeginConflictIntent`/`CompleteConflictIntent`) | Owns chain-tip bookkeeping during block building and reorgs: creates the coinbase UTXO and deletes it (with its child spends) on reorg, flags which transactions sit on the currently-longest chain (on a reorg, only the transactions it moved, not its whole mempool), unlocks unmined transactions reloaded at startup and conflict-locked parents when healing a stale intent, and drives the conflicting-transaction state machine (marking/unmarking double-spends and reversing that on reorg). | | **Legacy (p2p sync)** | `services/legacy/netsync/handle_block.go` | `SpendAndCreate`, `utxo.SetMinedMultiChunked` (→ `SetMinedMulti`) | The legacy Bitcoin-protocol block-sync path validates and applies blocks received from legacy peers the same way Block Validation does for the native protocol — same create/spend/set-mined writes, different ingestion path. | | **Alert System** | `services/alert/node.go` | `FreezeUTXOs`, `UnFreezeUTXOs`, `ReAssignUTXO` | Executes alert-network directives (court orders / network alerts) to freeze, unfreeze, or reassign specific UTXOs. | | **RPC** | `services/rpc/handlers.go` | `FreezeUTXOs`, `UnFreezeUTXOs`, `ReAssignUTXO` | Exposes the same freeze/unfreeze/reassign operations as operator-facing RPC calls (e.g. for manual intervention), independent of the Alert System's automated path. | diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 5feff3c091..caace35c5f 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1631,7 +1631,7 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // overtaken, the overtaken-tip repair must know about it, even if this attempt fails // and a retry sees onLongestChain=false. if onLongestChain && (len(unsetMined) == 0 || !unsetMined[0]) { - u.minedOnChain.track(*blockHash, blockHeaderMeta.ID, blockHeaderMeta.Height, false) + u.minedOnChain.track(*blockHash, blockHeaderMeta.ID, blockHeaderMeta.Height) } // add the transactions in this block to the block IDs in the utxo store diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index db044f89b6..e940769dd1 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -42,8 +42,11 @@ import ( // memory. After a restart the tracker is rebuilt from the chain without knowing which // blocks were re-marked, so a re-marked block that comes back onto the chain afterwards // keeps those txs flagged unmined until block assembly's next unmined-transaction load -// fixes them (fixUnminedSinceInconsistencies); and fork blocks deeper than -// tipRepairSeedForkDepth at the restart are not repaired. +// fixes them (fixUnminedSinceInconsistencies). That is the safe direction. The unsafe one +// remains for fork blocks deeper than tipRepairSeedForkDepth when the seed runs: the seed +// does not see them, so if one was overtaken with its repair unfinished, or with a +// setTxMined that cleared some of its txs and then failed, those txs keep looking mined +// while off the chain, and nothing else re-marks them. const ( // tipRepairWindow is how deep below the best block a tracked block is kept. A reorg @@ -111,10 +114,9 @@ func newMinedOnChainTracker() *minedOnChainTracker { return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock)} } -// track watches a block. With written=false a new on-chain write is starting: it clears the -// block's txs again, so the block gets a new gen and starts over. With written=true (the -// restart seed) a block already watched keeps its state. -func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32, written bool) { +// track watches a block whose on-chain write is starting: the write clears the block's txs +// again, so the block gets a new gen and starts over. +func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32) { if t == nil { return } @@ -122,13 +124,25 @@ func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32, writ t.mu.Lock() defer t.mu.Unlock() - b, ok := t.blocks[hash] - if ok && written { + t.nextGen++ + t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, failures: t.blocks[hash].failures} +} + +// seed watches a block found by the restart seed. A block already watched keeps its state. +func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32, written bool) { + if t == nil { + return + } + + t.mu.Lock() + defer t.mu.Unlock() + + if _, ok := t.blocks[hash]; ok { return } t.nextGen++ - t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, written: written, failures: b.failures} + t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, written: written} } // markWritten records that the block's setTxMined write has completed and reports whether @@ -306,7 +320,15 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo return errors.NewServiceError("[repairOvertakenTips] seed: failed to get block header %s", header.Hash().String(), err) } - u.trackMinedHeaders([]*model.BlockHeader{header}, []*model.BlockHeaderMeta{meta}) + if meta == nil || meta.Invalid { + continue + } + + // A block whose mined_set is not written may still have had txs cleared by a + // setTxMined that failed part way before the restart. Its retry, with + // onLongestChain=false once it is off the chain, leaves unmined_since alone, so it + // is watched unwritten and becomes repairable when that retry completes. + u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height, meta.MinedSet) } u.tipRepairSeeded = true @@ -320,7 +342,7 @@ func (u *BlockValidation) trackMinedHeaders(headers []*model.BlockHeader, metas continue } - u.minedOnChain.track(*headers[i].Hash(), meta.ID, meta.Height, true) + u.minedOnChain.seed(*headers[i].Hash(), meta.ID, meta.Height, true) } } @@ -332,7 +354,10 @@ func (u *BlockValidation) trackMinedHeaders(headers []*model.BlockHeader, metas // // The repair is serialised with this process's setMined worker only. Several block // validation instances writing setTxMined for the same chain would each run their own -// repair, unserialised against the others' writes. +// repair, unserialised against the others' writes. That race errs on the safe side: a +// mark(false) landing after another instance's setTxMined leaves the tx's block ids alone, +// so the tx is over-retained (flagged unmined while mined on the chain) until +// fixUnminedSinceInconsistencies clears it, never pruned while off the chain. func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient blockchain.ClientI) (more bool, err error) { u.tipRepairMu.Lock() defer u.tipRepairMu.Unlock() diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 4a894d9e07..9407cbbabf 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -30,15 +30,18 @@ import ( ) // overtakeTestStore wraps the UTXO store so a test can act at precise points: -// afterSetMined runs once, right after the first SetMinedMulti for blockID, and -// failMarkFalse makes the next N MarkTransactionsOnLongestChain(false) calls fail. +// afterSetMined runs once, right after the first SetMinedMulti for blockID, +// failMarkFalse makes the next N MarkTransactionsOnLongestChain(false) calls fail, and +// failAfterSetMined makes SetMinedMulti fail after its write landed: a setTxMined that +// cleared unmined_since and then failed, leaving the block's mined_set unwritten. type overtakeTestStore struct { utxostore.Store - blockID uint32 - once sync.Once - afterSetMined func() - failMarkFalse atomic.Int32 + blockID uint32 + once sync.Once + afterSetMined func() + failMarkFalse atomic.Int32 + failAfterSetMined atomic.Bool markedTrueMu sync.Mutex markedTrue []chainhash.Hash // every hash passed to MarkTransactionsOnLongestChain(true) @@ -46,6 +49,10 @@ type overtakeTestStore struct { func (s *overtakeTestStore) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { res, err := s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) + if err == nil && s.failAfterSetMined.Load() { + return nil, errors.NewStorageError("injected failure after the write") + } + if err == nil && s.afterSetMined != nil && minedBlockInfo.BlockID == s.blockID { s.once.Do(s.afterSetMined) } @@ -628,6 +635,39 @@ func TestRepairOvertakenTips_SeedAfterRestart(t *testing.T) { require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") } +// TestRepairOvertakenTips_PartialWriteSeededBeforeRetry: B's setTxMined clears its txs and +// then fails, so B's mined_set stays unwritten. The node restarts, the competing chain +// overtakes B, and the restarted process seeds the tracker before B's retry. The retry runs +// with onLongestChain=false, which leaves unmined_since alone, so the cleared txs would keep +// looking mined unless the seed tracks B although its mined_set is not written. +func TestRepairOvertakenTips_PartialWriteSeededBeforeRetry(t *testing.T) { + f := newOvertakeFixture(t) + + f.store.failAfterSetMined.Store(true) + require.Error(t, f.newBlockValidation(t).setTxMinedStatus(f.ctx, f.blockB.Hash())) + f.store.failAfterSetMined.Store(false) + + _, metaB, err := f.blockchainClient.GetBlockHeader(f.ctx, f.blockB.Hash()) + require.NoError(t, err) + require.False(t, metaB.MinedSet, "precondition: B's mined_set is not written") + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: the failed write cleared B's txs") + + blockC2 := f.overtakeB(t) + + restarted := f.newBlockValidation(t) + + require.NoError(t, restarted.setTxMinedStatus(f.ctx, blockC2.Hash())) + _, err = drainTipRepair(t, restarted) + require.NoError(t, err) + + require.NoError(t, restarted.setTxMinedStatus(f.ctx, f.blockB.Hash()), "B's retry, now off the longest chain") + _, err = drainTipRepair(t, restarted) + require.NoError(t, err) + + require.NotZero(t, f.unminedSince(t, f.bOnly), "a tx cleared by the failed write must be unmined again") + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") +} + // TestRepairOvertakenTips_SkipsPrunedTxs: a tx of the overtaken block that is no longer in // the store (pruned) must not stop the repair of the others. func TestRepairOvertakenTips_SkipsPrunedTxs(t *testing.T) { @@ -742,7 +782,7 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { tracker := newMinedOnChainTracker() hash := chainhash.HashH([]byte("b")) - tracker.track(hash, 1, 1, false) + tracker.track(hash, 1, 1) require.True(t, tracker.markWritten(hash)) gen := tracker.snapshot()[hash].gen @@ -752,7 +792,7 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { }) // setTxMined writes the block as on the longest chain again (its mined_set was cleared). - tracker.track(hash, 1, 1, false) + tracker.track(hash, 1, 1) b := tracker.snapshot()[hash] require.NotEqual(t, gen, b.gen, "a new write is a new generation; a repair in progress for the old one is abandoned") @@ -763,3 +803,186 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { tracker.update(hash, gen, func(e *minedOnChainBlock) { e.done = true }) require.False(t, tracker.snapshot()[hash].done, "an update for the old generation is ignored") } + +func TestMinedOnChainTracker_Nil(t *testing.T) { + var tracker *minedOnChainTracker + hash := chainhash.HashH([]byte("b")) + + tracker.track(hash, 1, 1) + tracker.seed(hash, 1, 1, true) + require.False(t, tracker.markWritten(hash)) + tracker.update(hash, 1, func(e *minedOnChainBlock) { e.done = true }) + tracker.drop(hash) + require.True(t, tracker.empty()) + require.Nil(t, tracker.snapshot()) +} + +// TestRepairOvertakenTips_DefaultSliceSize: an unset slice size falls back to the default. +func TestRepairOvertakenTips_DefaultSliceSize(t *testing.T) { + f := newOvertakeFixture(t) + f.tSettings.BlockValidation.TipRepairSliceSize = 0 + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly)) +} + +// TestRunTipRepairSlice_RetriesAfterError: a failed slice leaves the retry flag set, so the +// periodic tick asks again; the next good slice clears it. +func TestRunTipRepairSlice_RetriesAfterError(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + client := &failingGetBlockClient{ClientI: f.blockchainClient} + client.fail.Store(true) + + bv.runTipRepairSlice(f.ctx, client) + require.True(t, bv.tipRepairRetry.Load()) + + client.fail.Store(false) + + for i := 0; i < 10 && bv.tipRepairPending.Swap(false); i++ { + bv.runTipRepairSlice(f.ctx, client) + } + + bv.runTipRepairSlice(f.ctx, client) + require.False(t, bv.tipRepairRetry.Load()) + require.NotZero(t, f.unminedSince(t, f.bOnly)) +} + +// seedFailClient fails the one blockchain call named by failOn. +type seedFailClient struct { + blockchain.ClientI + failOn string +} + +func (c *seedFailClient) err(name string) error { + if c.failOn == name { + return errors.NewServiceError("injected %s failure", name) + } + + return nil +} + +func (c *seedFailClient) GetBestBlockHeader(ctx context.Context) (*model.BlockHeader, *model.BlockHeaderMeta, error) { + if err := c.err("GetBestBlockHeader"); err != nil { + return nil, nil, err + } + + return c.ClientI.GetBestBlockHeader(ctx) +} + +func (c *seedFailClient) GetBlockHeaders(ctx context.Context, hash *chainhash.Hash, n uint64) ([]*model.BlockHeader, []*model.BlockHeaderMeta, error) { + if err := c.err("GetBlockHeaders"); err != nil { + return nil, nil, err + } + + return c.ClientI.GetBlockHeaders(ctx, hash, n) +} + +func (c *seedFailClient) GetLastNBlocks(ctx context.Context, n int64, includeOrphans bool, fromHeight uint32) ([]*model.BlockInfo, error) { + if err := c.err("GetLastNBlocks"); err != nil { + return nil, err + } + + return c.ClientI.GetLastNBlocks(ctx, n, includeOrphans, fromHeight) +} + +func (c *seedFailClient) GetBlockHeader(ctx context.Context, hash *chainhash.Hash) (*model.BlockHeader, *model.BlockHeaderMeta, error) { + if err := c.err("GetBlockHeader"); err != nil { + return nil, nil, err + } + + return c.ClientI.GetBlockHeader(ctx, hash) +} + +func (c *seedFailClient) GetBlockHeaderIDs(ctx context.Context, hash *chainhash.Hash, n uint64) ([]uint32, error) { + if err := c.err("GetBlockHeaderIDs"); err != nil { + return nil, err + } + + return c.ClientI.GetBlockHeaderIDs(ctx, hash, n) +} + +// TestRepairOvertakenTips_SeedFailures: a blockchain call failing during the restart seed or +// the main-chain read fails the slice and leaves the seed to run again. +func TestRepairOvertakenTips_SeedFailures(t *testing.T) { + for _, failOn := range []string{"GetBestBlockHeader", "GetBlockHeaders", "GetLastNBlocks", "GetBlockHeader", "GetBlockHeaderIDs"} { + t.Run(failOn, func(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + // The seed must run in this slice. + bv.tipRepairSeeded = false + + _, err := bv.tipRepairSlice(f.ctx, &seedFailClient{ClientI: f.blockchainClient, failOn: failOn}) + require.Error(t, err) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the next slice repairs the block") + }) + } +} + +// farBestClient reports the best block as much higher than it is, so tracked blocks look +// older than the repair window. +type farBestClient struct { + blockchain.ClientI +} + +func (c *farBestClient) GetBestBlockHeader(ctx context.Context) (*model.BlockHeader, *model.BlockHeaderMeta, error) { + header, meta, err := c.ClientI.GetBestBlockHeader(ctx) + if meta != nil { + far := *meta + far.Height += 2 * tipRepairWindow + meta = &far + } + + return header, meta, err +} + +// TestRepairOvertakenTips_ExpiredBlockDropped: a block that left the window before its repair +// is dropped, not repaired. +func TestRepairOvertakenTips_ExpiredBlockDropped(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.blockchainClient = &farBestClient{ClientI: f.blockchainClient} + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.empty(), "the expired block is dropped") + require.Zero(t, f.unminedSince(t, f.bOnly), "and not repaired") +} + +// TestRepairOvertakenTips_InvalidBlockDropped: an invalidated block belongs to unsetMined, +// so the repair drops it. +func TestRepairOvertakenTips_InvalidBlockDropped(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + _, err := f.blockchainClient.InvalidateBlock(f.ctx, f.blockB.Hash()) + require.NoError(t, err) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + + _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, tracked, "the invalid block is dropped") +} From 70ab68ca65dd479051e0c1d0a418a432008abd48 Mon Sep 17 00:00:00 2001 From: Siggi Date: Tue, 27 Oct 2026 17:55:05 +0100 Subject: [PATCH 06/33] fix(blockvalidation): harden the overtaken-tip repair's subtree reads, stuck jobs and remine - Read a subtree the way setTxMined does: fall back to the .subtreeToCheck file and retry a failed read, so a block marked mined from a .subtreeToCheck file, or a read truncated while the file moves, no longer skips its txs for good. - Set aside a job that fails 10 steps in a row so it stops holding up the repair of every other block; it starts over once they have had their turn. - Remine only txs that still carry the block's id: a tx pruned and created again while the block was off the chain is in no block and stays unmined. - Run the reorg mark-scope tests on a real sqlitememory chain store instead of a mocked blockchain client. --- model/filter-not-on-longest-chain.go | 13 +- .../subtreeprocessor/reorg_mark_scope_test.go | 167 ++++++++++-------- .../blockvalidation/overtaken_tip_repair.go | 108 +++++++++-- .../settxmined_overtaken_test.go | 138 +++++++++++++++ 4 files changed, 325 insertions(+), 101 deletions(-) diff --git a/model/filter-not-on-longest-chain.go b/model/filter-not-on-longest-chain.go index 34e8063f95..171f5e5e34 100644 --- a/model/filter-not-on-longest-chain.go +++ b/model/filter-not-on-longest-chain.go @@ -3,6 +3,7 @@ package model import ( "context" "fmt" + "slices" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" @@ -141,11 +142,11 @@ func FilterTxsNotOnLongestChain(ctx context.Context, logger ulogger.Logger, tSet return notOnLongestChain, nil } -// FilterTxsUnmined returns the subset of hashes whose unmined_since is set: the txs a -// MarkTransactionsOnLongestChain(true) would change. Hashes missing from the UTXO store -// are skipped. -func FilterTxsUnmined(ctx context.Context, logger ulogger.Logger, tSettings *settings.Settings, utxoStore txMetaBatchReader, hashes []chainhash.Hash) ([]chainhash.Hash, error) { - items, err := batchReadTxMeta(ctx, logger, tSettings, utxoStore, hashes, fields.UnminedSince) +// FilterTxsUnmined returns the subset of hashes whose unmined_since is set and that are +// mined in blockID: the txs of that block a MarkTransactionsOnLongestChain(true) would +// change. Hashes missing from the UTXO store are skipped. +func FilterTxsUnmined(ctx context.Context, logger ulogger.Logger, tSettings *settings.Settings, utxoStore txMetaBatchReader, blockID uint32, hashes []chainhash.Hash) ([]chainhash.Hash, error) { + items, err := batchReadTxMeta(ctx, logger, tSettings, utxoStore, hashes, fields.UnminedSince, fields.BlockIDs) if err != nil { return nil, errors.NewProcessingError("[FilterTxsUnmined] error getting transactions from utxo store", err) } @@ -161,7 +162,7 @@ func FilterTxsUnmined(ctx context.Context, logger ulogger.Logger, tSettings *set return nil, errors.NewProcessingError("[FilterTxsUnmined] error getting transaction %s from utxo store", item.Hash.String(), item.Err) } - if item.Data != nil && item.Data.UnminedSince != 0 { + if item.Data != nil && item.Data.UnminedSince != 0 && slices.Contains(item.Data.BlockIDs, blockID) { unmined = append(unmined, item.Hash) } } diff --git a/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go index 30d208c6f8..8b86eb457b 100644 --- a/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go +++ b/services/blockassembly/subtreeprocessor/reorg_mark_scope_test.go @@ -14,13 +14,15 @@ import ( "github.com/bsv-blockchain/teranode/model" "github.com/bsv-blockchain/teranode/pkg/fileformat" "github.com/bsv-blockchain/teranode/services/blockchain" + "github.com/bsv-blockchain/teranode/settings" blob_memory "github.com/bsv-blockchain/teranode/stores/blob/memory" + blockchain_store "github.com/bsv-blockchain/teranode/stores/blockchain" + blockchainoptions "github.com/bsv-blockchain/teranode/stores/blockchain/options" utxostore "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/stores/utxo/fields" "github.com/bsv-blockchain/teranode/stores/utxo/sql" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util/test" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -136,6 +138,68 @@ func storeScopeTestSubtree(t *testing.T, ctx context.Context, blobStore *blob_me return key } +// newScopeTestChain returns a blockchain client over an in-memory SQLite chain store that +// holds only genesis. +func newScopeTestChain(t *testing.T, tSettings *settings.Settings) blockchain.ClientI { + t.Helper() + + chainStore, err := blockchain_store.NewStore(ulogger.TestLogger{}, &url.URL{Scheme: "sqlitememory"}, tSettings) + require.NoError(t, err) + + client, err := blockchain.NewLocalClient(ulogger.TestLogger{}, tSettings, chainStore, nil, nil) + require.NoError(t, err) + + return client +} + +// addScopeTestBlock mines a block on parent, adds it to the chain with its mined_set written, +// and returns it with the id the chain store gave it. +func addScopeTestBlock(t *testing.T, ctx context.Context, client blockchain.ClientI, parent *chainhash.Hash, height uint32, + coinbase *bt.Tx, subtrees ...*chainhash.Hash) *model.Block { + t.Helper() + + nBits, err := model.NewNBitFromString("207fffff") + require.NoError(t, err) + + header := &model.BlockHeader{ + Version: 1, + HashPrevBlock: parent, + HashMerkleRoot: coinbase.TxIDChainHash(), // distinct per block + Timestamp: uint32(time.Now().Unix()) + height, //nolint:gosec + Bits: *nBits, + } + + for ok, _, _ := header.HasMetTargetDifficulty(); !ok; ok, _, _ = header.HasMetTargetDifficulty() { + header.Nonce++ + } + + block, err := model.NewBlock(header, coinbase, subtrees, uint64(1+2*len(subtrees)), 0, height, 0) //nolint:gosec + require.NoError(t, err) + require.NoError(t, client.AddBlock(ctx, block, "test", blockchainoptions.WithSubtreesSet(true))) + // Block validation has finished with it: a reorg waits for blocks whose mined_set is unset. + require.NoError(t, client.SetBlockMinedSet(ctx, block.Hash())) + + _, meta, err := client.GetBlockHeader(ctx, block.Hash()) + require.NoError(t, err) + + block.ID = meta.ID + + return block +} + +// mineScopeTestTxs writes hashes as mined on the longest chain in block. +func mineScopeTestTxs(t *testing.T, ctx context.Context, store utxostore.Store, block *model.Block, hashes ...chainhash.Hash) { + t.Helper() + + ptrs := make([]*chainhash.Hash, len(hashes)) + for i := range hashes { + ptrs[i] = &hashes[i] + } + + _, err := store.SetMinedMulti(ctx, ptrs, utxostore.MinedBlockInfo{BlockID: block.ID, BlockHeight: block.Height, OnLongestChain: true}) + require.NoError(t, err) +} + // TestReorgBlocks_MarkFalseScope pins issue 2051: a reorg marks not-on-longest-chain only // the txs whose chain state it changes (moved back and not moved forward), never the whole // block assembly mempool. @@ -156,11 +220,10 @@ func TestReorgBlocks_MarkFalseScope(t *testing.T) { store := newScopeMarkRecorder(sqlStore) blobStore := blob_memory.New() + chain := newScopeTestChain(t, tSettings) - const blockAID = uint32(10) - - a1 := createScopeTestTx(t, ctx, store, 0, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: blockAID, BlockHeight: 2, OnLongestChain: true})) - shared := createScopeTestTx(t, ctx, store, 1, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: blockAID, BlockHeight: 2, OnLongestChain: true})) + a1 := createScopeTestTx(t, ctx, store, 0) + shared := createScopeTestTx(t, ctx, store, 1) b1 := createScopeTestTx(t, ctx, store, 2) m1 := createScopeTestTx(t, ctx, store, 3) m2 := createScopeTestTx(t, ctx, store, 4) @@ -178,41 +241,12 @@ func TestReorgBlocks_MarkFalseScope(t *testing.T) { _, err = sqlStore.Create(ctx, coinbaseTx3, 2) require.NoError(t, err) - blockA := &model.Block{ - ID: blockAID, - Height: 2, - CoinbaseTx: coinbaseTx2, - Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, a1, shared)}, - Header: &model.BlockHeader{ - Version: 1, - HashPrevBlock: prevBlockHeader.Hash(), - HashMerkleRoot: &chainhash.Hash{}, - Timestamp: 3000000001, - Bits: model.NBit{}, - Nonce: 501, - }, - } - - blockB := &model.Block{ - ID: blockAID + 1, - Height: 2, - CoinbaseTx: coinbaseTx3, - Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, shared, b1)}, - Header: &model.BlockHeader{ - Version: 1, - HashPrevBlock: prevBlockHeader.Hash(), - HashMerkleRoot: &chainhash.Hash{}, - Timestamp: 3000000002, - Bits: model.NBit{}, - Nonce: 502, - }, - } - - mockBlockchainClient := &blockchain.Mock{} - mockBlockchainClient.On("GetBlocksMinedNotSet", mock.Anything).Return([]*model.Block{}, nil) - mockBlockchainClient.On("SetBlockProcessedAt", mock.Anything, mock.AnythingOfType("*chainhash.Hash"), mock.AnythingOfType("[]bool")).Return(nil) - mockBlockchainClient.On("GetBlockHeader", mock.Anything, mock.Anything).Return(prevBlockHeader, &model.BlockHeaderMeta{}, nil) - mockBlockchainClient.On("GetBlockIsMined", mock.Anything, mock.Anything).Return(true, nil) + // A and B compete on genesis. A was the tip and mined a1 and shared; B also mines shared, + // and b1, which as a side-chain tx is still unmined. + genesis := tSettings.ChainCfgParams.GenesisHash + blockA := addScopeTestBlock(t, ctx, chain, genesis, 1, coinbaseTx2, storeScopeTestSubtree(t, ctx, blobStore, a1, shared)) + blockB := addScopeTestBlock(t, ctx, chain, genesis, 1, coinbaseTx3, storeScopeTestSubtree(t, ctx, blobStore, shared, b1)) + mineScopeTestTxs(t, ctx, store, blockA, a1, shared) newSubtreeChan := make(chan NewSubtreeRequest, 10) go func() { @@ -224,7 +258,7 @@ func TestReorgBlocks_MarkFalseScope(t *testing.T) { }() t.Cleanup(func() { close(newSubtreeChan) }) - stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, mockBlockchainClient, store, newSubtreeChan) + stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, chain, store, newSubtreeChan) require.NoError(t, err) stp.Start(ctx) @@ -294,48 +328,27 @@ func TestReorgBlocks_InvalidMoveBackChecksOnlyItsOwnTxs(t *testing.T) { store := newScopeMarkRecorder(sqlStore) blobStore := blob_memory.New() + chain := newScopeTestChain(t, tSettings) - const ( - invalidBlockID = uint32(20) - validBlockID = uint32(5) - ) - - a1 := createScopeTestTx(t, ctx, store, 0, utxostore.WithMinedBlockInfo(utxostore.MinedBlockInfo{BlockID: invalidBlockID, BlockHeight: 2, OnLongestChain: true})) - dup := createScopeTestTx(t, ctx, store, 1, utxostore.WithMinedBlockInfo( - utxostore.MinedBlockInfo{BlockID: validBlockID, BlockHeight: 1, OnLongestChain: true}, - utxostore.MinedBlockInfo{BlockID: invalidBlockID, BlockHeight: 2, OnLongestChain: true}, - )) + a1 := createScopeTestTx(t, ctx, store, 0) + dup := createScopeTestTx(t, ctx, store, 1) m1 := createScopeTestTx(t, ctx, store, 2) _, err = sqlStore.Create(ctx, coinbaseTx2, 2) require.NoError(t, err) - blockA := &model.Block{ - ID: invalidBlockID, - Height: 2, - CoinbaseTx: coinbaseTx2, - Subtrees: []*chainhash.Hash{storeScopeTestSubtree(t, ctx, blobStore, a1, dup)}, - Header: &model.BlockHeader{ - Version: 1, - HashPrevBlock: prevBlockHeader.Hash(), - HashMerkleRoot: &chainhash.Hash{}, - Timestamp: 3000000011, - Bits: model.NBit{}, - Nonce: 511, - }, - } + // genesis -> V -> A. V mined dup; A mined a1 and dup again, and is then invalidated, so + // V is the tip. + blockV := addScopeTestBlock(t, ctx, chain, tSettings.ChainCfgParams.GenesisHash, 1, coinbaseTx3) + blockA := addScopeTestBlock(t, ctx, chain, blockV.Hash(), 2, coinbaseTx2, storeScopeTestSubtree(t, ctx, blobStore, a1, dup)) + mineScopeTestTxs(t, ctx, store, blockV, dup) + mineScopeTestTxs(t, ctx, store, blockA, a1, dup) - mockBlockchainClient := &blockchain.Mock{} - mockBlockchainClient.On("GetBlocksMinedNotSet", mock.Anything).Return([]*model.Block{}, nil) - mockBlockchainClient.On("SetBlockProcessedAt", mock.Anything, mock.AnythingOfType("*chainhash.Hash"), mock.AnythingOfType("[]bool")).Return(nil) - mockBlockchainClient.On("GetBlockHeader", mock.Anything, mock.Anything).Return(prevBlockHeader, &model.BlockHeaderMeta{Invalid: true}, nil) - mockBlockchainClient.On("GetBlockHeaders", mock.Anything, mock.Anything, mock.Anything). - Return([]*model.BlockHeader{}, []*model.BlockHeaderMeta{{ID: validBlockID}}, nil) - // Only reached for a tx in neither the invalid block alone nor a recent block, i.e. a - // mempool tx that should never have been checked. - mockBlockchainClient.On("CheckBlockIsInCurrentChain", mock.Anything, mock.Anything).Return(false, nil) - mockBlockchainClient.On("GetBlock", mock.Anything, mock.Anything).Return(&model.Block{Header: prevBlockHeader, Height: 1}, nil) - mockBlockchainClient.On("GetBlockIsMined", mock.Anything, mock.Anything).Return(true, nil) + _, err = chain.InvalidateBlock(ctx, blockA.Hash()) + require.NoError(t, err) + // Invalidating clears A's mined_set for block validation's unset-mined pass, and the + // reorg waits for that pass to finish. + require.NoError(t, chain.SetBlockMinedSet(ctx, blockA.Hash())) newSubtreeChan := make(chan NewSubtreeRequest, 10) go func() { @@ -347,7 +360,7 @@ func TestReorgBlocks_InvalidMoveBackChecksOnlyItsOwnTxs(t *testing.T) { }() t.Cleanup(func() { close(newSubtreeChan) }) - stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, mockBlockchainClient, store, newSubtreeChan) + stp, err := NewSubtreeProcessor(ctx, ulogger.TestLogger{}, tSettings, blobStore, chain, store, newSubtreeChan) require.NoError(t, err) stp.Start(ctx) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index e940769dd1..171bb0fe37 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "sync" + "time" "github.com/bsv-blockchain/go-bt/v2/chainhash" subtreepkg "github.com/bsv-blockchain/go-subtree" @@ -67,6 +68,10 @@ const ( // defaultTipRepairSliceSize is used when BlockValidation.TipRepairSliceSize is unset. defaultTipRepairSliceSize = 250_000 + + // tipRepairSubtreeReadAttempts and tipRepairSubtreeReadBackoff match Block.GetSubtrees. + tipRepairSubtreeReadAttempts = 3 + tipRepairSubtreeReadBackoff = 100 * time.Millisecond ) // tipRepairExpired reports whether a tracked block at height has left the window. Each slice @@ -100,6 +105,9 @@ type minedOnChainBlock struct { // the chain; it is cleared when the block is seen back on the chain. done bool failures int // permanent failures so far + // deferred is set when the block's job kept failing and was set aside, so other + // blocks are repaired first. + deferred bool } // minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker @@ -226,8 +234,7 @@ const ( // unmarkJob re-marks the txs of a block that left the chain and are not mined on it. unmarkJob tipRepairJobKind = iota // remineJob marks the txs of a re-marked block mined again, now that it is back on the - // chain. Every one of them carries the block's id, so clearing unmined_since is right - // for all of them. + // chain, for those that still carry the block's id. remineJob ) @@ -245,6 +252,8 @@ type tipRepairJob struct { current []subtreepkg.Node marked int total int + // failures counts the consecutive failed steps. + failures int } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -414,28 +423,55 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b return true, nil } - return true, u.jobStep(ctx, blockchainClient, job, onMainChain) + if err = u.jobStep(ctx, blockchainClient, job, onMainChain); err == nil { + job.failures = 0 + return true, nil + } + + // A job that keeps failing is set aside, so it does not hold up the other blocks; + // it starts over once they have had their turn. + if job.failures++; job.failures >= tipRepairMaxAttempts { + u.logger.Warnf("[repairOvertakenTips][%s] setting the repair aside after %d failed steps, other blocks go first", job.hash.String(), job.failures) + u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.deferred = true }) + u.tipRepairJob = nil + } + + return true, err } + // Blocks set aside after failing come last. + for _, deferred := range []bool{false, true} { + if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, deferred); started { + return more, err + } + } + + return false, nil +} + +// startNextJob starts a job on the next block that needs one, among the blocks whose deferred +// flag matches, and reports whether it found one. +func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blockchain.ClientI, tracked map[chainhash.Hash]minedOnChainBlock, + onMainChain map[uint32]bool, deferred bool) (more, started bool, err error) { // A re-marked block back on the chain first: its txs look unmined while mined. for hash, b := range tracked { - if b.stamped && onMainChain[b.id] { - return true, u.startJob(ctx, blockchainClient, remineJob, hash, b) + if b.deferred == deferred && b.stamped && onMainChain[b.id] { + return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b) } } for hash, b := range tracked { - if !b.written || b.done || onMainChain[b.id] { + if b.deferred != deferred || !b.written || b.done || onMainChain[b.id] { continue } _, meta, err := blockchainClient.GetBlockHeader(ctx, &hash) if err != nil { - return true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block header", hash.String(), err)) + return true, true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block header", hash.String(), err)) } if meta == nil { - return true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] block header meta is nil", hash.String())) + return true, true, u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] block header meta is nil", hash.String())) } if meta.Invalid { @@ -444,10 +480,10 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b continue } - return true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) + return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) } - return false, nil + return false, false, nil } // mainChainIDsLocked returns the ids of the tipRepairWindow blocks ending at the best block, @@ -504,9 +540,13 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch return u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block", hash.String(), err)) } - if kind == remineJob { - u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.done = false }) - } + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { + e.deferred = false + + if kind == remineJob { + e.done = false + } + }) u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: block.Subtrees} @@ -571,8 +611,9 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha job.marked += len(notOnLongestChain) case remineJob: - // Only the txs that were stamped need writing. - unmined, err := model.FilterTxsUnmined(ctx, u.logger, u.settings, u.utxoStore, slice) + // Only the txs that were stamped need writing, and only those still in this block: a + // tx pruned and created again while the block was off the chain is in no block. + unmined, err := model.FilterTxsUnmined(ctx, u.logger, u.settings, u.utxoStore, job.id, slice) if err != nil { return errors.NewProcessingError("[repairOvertakenTips][%s] error reading txs of block back on the longest chain", job.hash.String(), err) } @@ -645,9 +686,41 @@ func (u *BlockValidation) repairFailed(hash chainhash.Hash, gen uint64, err erro return err } -// loadSubtreeNodes returns the nodes of one subtree. +// loadSubtreeNodes returns the nodes of one subtree, read like setTxMined reads it +// (Block.GetSubtrees): from the .subtree file, or the .subtreeToCheck file when that is +// missing. A read that fails, for example truncated while another service moves the file, is +// tried again; one that still cannot be deserialized is a corrupt subtree. func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { + var err error + + for attempt := 0; attempt < tipRepairSubtreeReadAttempts; attempt++ { + if attempt > 0 { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(tipRepairSubtreeReadBackoff): + } + } + + var nodes []subtreepkg.Node + if nodes, err = u.readSubtreeNodes(ctx, subtreeHash); err == nil { + return nodes, nil + } + + if errors.Is(err, errors.ErrNotFound) { + return nil, err + } + } + + return nil, errors.NewSubtreeNotFoundError("failed to read subtree %s", subtreeHash.String(), err) +} + +func (u *BlockValidation) readSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { reader, err := u.subtreeStore.GetIoReader(ctx, subtreeHash[:], fileformat.FileTypeSubtree) + if errors.Is(err, errors.ErrNotFound) { + reader, err = u.subtreeStore.GetIoReader(ctx, subtreeHash[:], fileformat.FileTypeSubtreeToCheck) + } + if err != nil { return nil, err } @@ -658,8 +731,7 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha subtree, err := subtreepkg.NewSubtreeFromReader(bufio.NewReader(reader)) if err != nil { - // A corrupt subtree cannot be read on a retry either. - return nil, errors.NewSubtreeNotFoundError("failed to deserialize subtree %s", subtreeHash.String(), err) + return nil, errors.NewProcessingError("failed to deserialize subtree %s", subtreeHash.String(), err) } return subtree.Nodes, nil diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 9407cbbabf..5d29131ac3 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -2,6 +2,7 @@ package blockvalidation import ( "context" + "io" "net/url" "slices" "sync" @@ -20,6 +21,7 @@ import ( "github.com/bsv-blockchain/teranode/services/subtreevalidation" "github.com/bsv-blockchain/teranode/settings" "github.com/bsv-blockchain/teranode/stores/blob" + blobopts "github.com/bsv-blockchain/teranode/stores/blob/options" blockchain_store "github.com/bsv-blockchain/teranode/stores/blockchain" blockchainoptions "github.com/bsv-blockchain/teranode/stores/blockchain/options" utxostore "github.com/bsv-blockchain/teranode/stores/utxo" @@ -106,6 +108,7 @@ type overtakeFixture struct { nBits *model.NBit blockB *model.Block blockBID uint32 + bOnlyTx *bt.Tx bOnly *chainhash.Hash shared *chainhash.Hash } @@ -173,6 +176,7 @@ func newOvertakeFixture(t *testing.T) *overtakeFixture { nBits: nBits, blockB: blockB, blockBID: metaB.ID, + bOnlyTx: bOnlyTx, bOnly: bOnlyTx.TxIDChainHash(), shared: sharedTx.TxIDChainHash(), } @@ -986,3 +990,137 @@ func TestRepairOvertakenTips_InvalidBlockDropped(t *testing.T) { _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] require.False(t, tracked, "the invalid block is dropped") } + +// TestRepairOvertakenTips_SubtreeToCheckOnly: setTxMined reads a subtree from its +// .subtreeToCheck file when the .subtree file is not there, so the repair must too, or a +// block marked mined that way is never repaired. +func TestRepairOvertakenTips_SubtreeToCheckOnly(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + subtreeHash := f.blockB.Subtrees[0] + subtreeBytes, err := f.subtreeStore.Get(f.ctx, subtreeHash[:], fileformat.FileTypeSubtree) + require.NoError(t, err) + require.NoError(t, f.subtreeStore.Set(f.ctx, subtreeHash[:], fileformat.FileTypeSubtreeToCheck, subtreeBytes)) + require.NoError(t, f.subtreeStore.Del(f.ctx, subtreeHash[:], fileformat.FileTypeSubtree)) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is read from its .subtreeToCheck file") +} + +// truncatingSubtreeStore returns a truncated reader for the next truncate reads: a subtree +// read while another service moves the file. +type truncatingSubtreeStore struct { + blob.Store + truncate atomic.Int32 +} + +func (s *truncatingSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { + reader, err := s.Store.GetIoReader(ctx, key, fileType, opts...) + if err != nil || s.truncate.Load() <= 0 { + return reader, err + } + + s.truncate.Add(-1) + + return struct { + io.Reader + io.Closer + }{io.LimitReader(reader, 10), reader}, nil +} + +// TestRepairOvertakenTips_TruncatedSubtreeReadRetried: a truncated subtree read must be +// retried, not taken for a subtree that is gone, which would skip its txs for good. +func TestRepairOvertakenTips_TruncatedSubtreeReadRetried(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + store := &truncatingSubtreeStore{Store: f.subtreeStore} + store.truncate.Store(1) + bv.subtreeStore = store + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is read again and repaired") +} + +// TestRepairOvertakenTips_StuckJobYields: a job that keeps failing must not hold up the +// repair of other blocks. +func TestRepairOvertakenTips_StuckJobYields(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + // A second overtaken block, written mined while it was on the chain. It has no txs, so + // its repair finishes in one step. + time.Sleep(10 * time.Millisecond) + + fork := coinbaseOnlyBlock(t, f.tSettings.ChainCfgParams.GenesisHash, f.nBits, 1, 0xf1) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, fork, "test", blockchainoptions.WithSubtreesSet(true))) + + _, forkMeta, err := f.blockchainClient.GetBlockHeader(f.ctx, fork.Hash()) + require.NoError(t, err) + require.False(t, f.onChain(t, forkMeta.ID)) + + f.store.failMarkFalse.Store(1 << 30) + + // B's job starts first and gets stuck. + _, err = bv.tipRepairSlice(f.ctx, bv.blockchainClient) + require.NoError(t, err) + require.NotNil(t, bv.tipRepairJob) + require.Equal(t, *f.blockB.Hash(), bv.tipRepairJob.hash) + + bv.minedOnChain.track(*fork.Hash(), forkMeta.ID, forkMeta.Height) + require.True(t, bv.minedOnChain.markWritten(*fork.Hash())) + + for i := 0; i < 3*tipRepairMaxAttempts; i++ { + _, _ = bv.tipRepairSlice(f.ctx, bv.blockchainClient) + } + + require.True(t, bv.minedOnChain.snapshot()[*fork.Hash()].done, "the other block is repaired while B's job keeps failing") + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "B is not done") + + f.store.failMarkFalse.Store(0) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired once the failure clears") +} + +// TestRepairOvertakenTips_RemineSkipsTxNotInBlock: a tx of a re-marked block that was pruned +// and created again unmined while the block was off the chain no longer carries the block's +// id. Marking it mined when the block comes back would leave it looking mined in no block. +func TestRepairOvertakenTips_RemineSkipsTxNotInBlock(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped, "precondition: B's txs were re-marked") + + require.NoError(t, f.store.Store.Delete(f.ctx, f.bOnly)) + _, _, err = f.store.Store.SpendAndCreate(f.ctx, f.bOnlyTx, 1, utxostore.WithCreateOnly()) + require.NoError(t, err) + + f.extendB(t) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + + m, err := f.store.Store.Get(f.ctx, f.bOnly, fields.UnminedSince, fields.BlockIDs) + require.NoError(t, err) + require.Empty(t, m.BlockIDs, "precondition: the re-created tx is in no block") + require.NotZero(t, m.UnminedSince, "a tx in no block must stay unmined") +} From d8fb981a8383afc5c252e5dc869ff9cce1c151e8 Mon Sep 17 00:00:00 2001 From: Siggi Date: Tue, 27 Oct 2026 23:29:36 +0100 Subject: [PATCH 07/33] fix(blockvalidation): repair partially written blocks, retry transient subtree read errors The overtaken-tip repair now repairs a tracked block whose setTxMined write failed part way, instead of waiting for a retry to complete it; a retry that keeps failing left the cleared txs looking mined while off the chain. The restart seed watches main-chain blocks whose mined_set is unwritten as well. A completed write has a finished repair run again. loadSubtreeNodes only treats a subtree that cannot be deserialized as gone. A blob-store error that outlasts the read attempts is returned as is, so the step is retried rather than the subtree skipped for good. Updates the MarkTransactionsOnLongestChain signature in the utxo store reference. --- docs/references/stores/utxo_reference.md | 4 +- services/blockvalidation/BlockValidation.go | 11 +-- .../blockvalidation/overtaken_tip_repair.go | 51 +++++----- .../settxmined_overtaken_test.go | 92 +++++++++++++++++-- 4 files changed, 119 insertions(+), 39 deletions(-) diff --git a/docs/references/stores/utxo_reference.md b/docs/references/stores/utxo_reference.md index 36c6482b1f..64002410c7 100644 --- a/docs/references/stores/utxo_reference.md +++ b/docs/references/stores/utxo_reference.md @@ -256,7 +256,9 @@ type Store interface { // MarkTransactionsOnLongestChain marks transactions as being on the longest chain or not. // When onLongestChain is true, the unminedSince field is unset (transaction is mined). // When onLongestChain is false, the unminedSince field is set to the current block height. - MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool) error + // WithSkipMissing() makes a record missing from the store a logged skip instead of an + // error for onLongestChain=false. + MarkTransactionsOnLongestChain(ctx context.Context, txHashes []chainhash.Hash, onLongestChain bool, opts ...MarkLongestChainOption) error // SetBlockHeight updates the current block height in the store. // diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index caace35c5f..bd67ef5e57 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1657,8 +1657,8 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // provenance is attached; the p2p consumer falls back to its peer map. return u.markBlockAsInvalid(ctx, block, "contains transactions already on our chain: "+err.Error(), "", "") } else if errors.Is(err, errors.ErrBlockParentNotMined) { - // Another call for this block is in flight. Its tracking stays: that write - // marks it written when it completes. + // Another call for this block is in flight. Its tracking stays, and that write + // has the block repaired again when it completes. u.logger.Warnf("[setTxMined][%s] skipping, already in progress of setMined for block", block.Hash().String()) return nil } @@ -1666,10 +1666,9 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain return errors.NewProcessingError("[setTxMined][%s] error updating tx mined status", block.Hash().String(), err) } - // The write is complete, whichever onLongestChain this attempt used: a block tracked by - // an earlier attempt (whose clears may have landed) becomes eligible for the repair now. - // Ask for a slice: a chain change that happened while the write was in flight would - // otherwise wait for the next one. + // The write is complete, whichever onLongestChain this attempt used. Ask for a slice: a + // chain change that happened while the write was in flight would otherwise wait for the + // next one. if u.minedOnChain.markWritten(*blockHash) { u.signalTipRepair() } diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 171bb0fe37..db15b5c1b1 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -88,16 +88,14 @@ func isPermanentRepairError(err error) bool { errors.Is(err, errors.ErrBlobNotFound) || errors.Is(err, errors.ErrNotFound) } -// minedOnChainBlock is a block setTxMined wrote with onLongestChain=true. +// minedOnChainBlock is a block whose txs setTxMined may have marked mined on the longest +// chain, in a write that completed or one that failed part way. type minedOnChainBlock struct { id uint32 height uint32 // gen changes whenever a new on-chain write of the block starts. A repair in progress // for an older gen is abandoned: the new write cleared the txs again. gen uint64 - // written is set once the block's setTxMined write has completed. A repair only starts - // on a written block. - written bool // stamped is set once any of the block's txs were re-marked not on the longest chain, // so they need marking mined again if the block comes back onto the chain. stamped bool @@ -137,7 +135,7 @@ func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32) { } // seed watches a block found by the restart seed. A block already watched keeps its state. -func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32, written bool) { +func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32) { if t == nil { return } @@ -150,11 +148,12 @@ func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32, writt } t.nextGen++ - t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, written: written} + t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen} } -// markWritten records that the block's setTxMined write has completed and reports whether -// the block is tracked, in which case the caller asks for a repair slice. +// markWritten records that a setTxMined write of the block has completed and reports +// whether the block is tracked, in which case the caller asks for a repair slice. The write +// may have cleared txs after a repair read them, so a finished repair runs again. func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { if t == nil { return false @@ -168,7 +167,7 @@ func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { return false } - b.written = true + b.done = false t.blocks[hash] = b return true @@ -291,8 +290,8 @@ func (u *BlockValidation) runTipRepairSlice(ctx context.Context, blockchainClien } // seedMinedOnChainTrackerLocked rebuilds the in-memory tracker after a restart: the -// main-chain blocks inside the window that are mined, and the mined fork blocks within -// tipRepairSeedForkDepth of the best height, which may have been the tip before the restart. +// main-chain blocks inside the window, and the fork blocks within tipRepairSeedForkDepth of +// the best height, which may have been the tip before the restart. func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { bestHeader, bestMeta, err := blockchainClient.GetBestBlockHeader(ctx) if err != nil { @@ -333,11 +332,10 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo continue } - // A block whose mined_set is not written may still have had txs cleared by a - // setTxMined that failed part way before the restart. Its retry, with - // onLongestChain=false once it is off the chain, leaves unmined_since alone, so it - // is watched unwritten and becomes repairable when that retry completes. - u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height, meta.MinedSet) + // A block whose mined_set is not written is watched too: a setTxMined that failed + // part way before the restart may have cleared some of its txs, and its retry, + // with onLongestChain=false once it is off the chain, leaves unmined_since alone. + u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height) } u.tipRepairSeeded = true @@ -347,11 +345,13 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo func (u *BlockValidation) trackMinedHeaders(headers []*model.BlockHeader, metas []*model.BlockHeaderMeta) { for i, meta := range metas { - if i >= len(headers) || meta == nil || !meta.MinedSet || meta.Invalid { + if i >= len(headers) || meta == nil || meta.Invalid { continue } - u.minedOnChain.seed(*headers[i].Hash(), meta.ID, meta.Height, true) + // A block whose mined_set is not written too: a setTxMined that failed part way + // may have cleared some of its txs. + u.minedOnChain.seed(*headers[i].Hash(), meta.ID, meta.Height) } } @@ -395,7 +395,7 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b for hash, b := range tracked { switch { case tipRepairExpired(b.height, bestHeight): - if b.written && !onMainChain[b.id] && !b.done { + if !onMainChain[b.id] && !b.done { u.logger.Warnf("[repairOvertakenTips][%s] block left the repair window before its repair finished; some of its txs may still look mined", hash.String()) } @@ -461,7 +461,7 @@ func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blo } for hash, b := range tracked { - if b.deferred != deferred || !b.written || b.done || onMainChain[b.id] { + if b.deferred != deferred || b.done || onMainChain[b.id] { continue } @@ -689,7 +689,8 @@ func (u *BlockValidation) repairFailed(hash chainhash.Hash, gen uint64, err erro // loadSubtreeNodes returns the nodes of one subtree, read like setTxMined reads it // (Block.GetSubtrees): from the .subtree file, or the .subtreeToCheck file when that is // missing. A read that fails, for example truncated while another service moves the file, is -// tried again; one that still cannot be deserialized is a corrupt subtree. +// tried again; one that still cannot be deserialized is a corrupt subtree, and any other +// error is left for the next step to retry. func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { var err error @@ -712,7 +713,13 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha } } - return nil, errors.NewSubtreeNotFoundError("failed to read subtree %s", subtreeHash.String(), err) + // Only a subtree that still cannot be deserialized is corrupt. Any other error, such + // as a blob store that is briefly down, is returned as it is, so the step is retried. + if errors.Is(err, errors.ErrProcessing) { + return nil, errors.NewSubtreeNotFoundError("failed to read subtree %s", subtreeHash.String(), err) + } + + return nil, err } func (u *BlockValidation) readSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 5d29131ac3..3b9384cf93 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -802,7 +802,6 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { require.NotEqual(t, gen, b.gen, "a new write is a new generation; a repair in progress for the old one is abandoned") require.False(t, b.done, "the new write clears the txs again, so the block needs repairing again") require.False(t, b.stamped) - require.False(t, b.written, "and must not be repaired until the new write completes") tracker.update(hash, gen, func(e *minedOnChainBlock) { e.done = true }) require.False(t, tracker.snapshot()[hash].done, "an update for the old generation is ignored") @@ -813,7 +812,7 @@ func TestMinedOnChainTracker_Nil(t *testing.T) { hash := chainhash.HashH([]byte("b")) tracker.track(hash, 1, 1) - tracker.seed(hash, 1, 1, true) + tracker.seed(hash, 1, 1) require.False(t, tracker.markWritten(hash)) tracker.update(hash, 1, func(e *minedOnChainBlock) { e.done = true }) tracker.drop(hash) @@ -1060,6 +1059,14 @@ func TestRepairOvertakenTips_StuckJobYields(t *testing.T) { require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) _ = f.overtakeB(t) + f.store.failMarkFalse.Store(1 << 30) + + // B's job starts first and gets stuck. + _, err := bv.tipRepairSlice(f.ctx, bv.blockchainClient) + require.NoError(t, err) + require.NotNil(t, bv.tipRepairJob) + require.Equal(t, *f.blockB.Hash(), bv.tipRepairJob.hash) + // A second overtaken block, written mined while it was on the chain. It has no txs, so // its repair finishes in one step. time.Sleep(10 * time.Millisecond) @@ -1071,14 +1078,6 @@ func TestRepairOvertakenTips_StuckJobYields(t *testing.T) { require.NoError(t, err) require.False(t, f.onChain(t, forkMeta.ID)) - f.store.failMarkFalse.Store(1 << 30) - - // B's job starts first and gets stuck. - _, err = bv.tipRepairSlice(f.ctx, bv.blockchainClient) - require.NoError(t, err) - require.NotNil(t, bv.tipRepairJob) - require.Equal(t, *f.blockB.Hash(), bv.tipRepairJob.hash) - bv.minedOnChain.track(*fork.Hash(), forkMeta.ID, forkMeta.Height) require.True(t, bv.minedOnChain.markWritten(*fork.Hash())) @@ -1124,3 +1123,76 @@ func TestRepairOvertakenTips_RemineSkipsTxNotInBlock(t *testing.T) { require.Empty(t, m.BlockIDs, "precondition: the re-created tx is in no block") require.NotZero(t, m.UnminedSince, "a tx in no block must stay unmined") } + +// TestRepairOvertakenTips_PartialWriteRetryKeepsFailing: B's setTxMined clears its txs and +// then fails, and every retry fails too, so B's mined_set is never written. Once the +// competing chain overtakes B, the cleared txs must be re-marked without waiting for a retry +// to succeed, whether the process kept B's tracking or restarted and seeded it. +func TestRepairOvertakenTips_PartialWriteRetryKeepsFailing(t *testing.T) { + for _, restart := range []bool{false, true} { + t.Run(map[bool]string{false: "in process", true: "after restart"}[restart], func(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + f.store.failAfterSetMined.Store(true) + require.Error(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: the failed write cleared B's txs") + + blockC2 := f.overtakeB(t) + + if restart { + bv = f.newBlockValidation(t) + + f.store.failAfterSetMined.Store(false) + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockC2.Hash())) + f.store.failAfterSetMined.Store(true) + } + + require.Error(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash()), "B's retry, off the longest chain, fails again") + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + + require.NotZero(t, f.unminedSince(t, f.bOnly), "a tx cleared by the failed write must be unmined again") + require.Zero(t, f.unminedSince(t, f.shared), "a tx also mined on the winning chain must stay mined") + }) + } +} + +// failingSubtreeStore fails the next fail subtree reads with a storage error: a blob store +// that is briefly unavailable. +type failingSubtreeStore struct { + blob.Store + fail atomic.Int32 +} + +func (s *failingSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { + if s.fail.Add(-1) >= 0 { + return nil, errors.NewStorageError("injected subtree store failure") + } + + return s.Store.GetIoReader(ctx, key, fileType, opts...) +} + +// TestRepairOvertakenTips_TransientSubtreeReadErrorRetried: a subtree store that stays +// unavailable through every read attempt of a step must not get the subtree skipped as gone; +// the step is retried once the store is back. +func TestRepairOvertakenTips_TransientSubtreeReadErrorRetried(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + store := &failingSubtreeStore{Store: f.subtreeStore} + store.fail.Store(tipRepairSubtreeReadAttempts) + bv.subtreeStore = store + + _, err := drainTipRepair(t, bv) + require.Error(t, err, "the step fails while the store is down") + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is not skipped") + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once the store is back") +} From a4cfefac54baecbff76f954a609f9021c8e9f2bf Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 12:32:27 +0100 Subject: [PATCH 08/33] fix(blockvalidation): retry a subtree whose reads stay short instead of skipping it A short read looks the same as a corrupt subtree file, so mapping a deserialize failure that outlasts the read attempts to SubtreeNotFound let a run of short reads skip the subtree for good, leaving its txs looking mined while off the chain. The error is now returned for the next step to retry; a job that keeps failing is set aside. Also removes a race in TestRepairOvertakenTips_RunsOnTheSetMinedWorker: the test wrote the competing block's tx straight to the store after the chain flipped, where the live worker's repair slice could interleave with it. --- .../blockvalidation/overtaken_tip_repair.go | 12 ++---- .../settxmined_overtaken_test.go | 39 ++++++++++++++++++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index db15b5c1b1..257b356219 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -689,8 +689,7 @@ func (u *BlockValidation) repairFailed(hash chainhash.Hash, gen uint64, err erro // loadSubtreeNodes returns the nodes of one subtree, read like setTxMined reads it // (Block.GetSubtrees): from the .subtree file, or the .subtreeToCheck file when that is // missing. A read that fails, for example truncated while another service moves the file, is -// tried again; one that still cannot be deserialized is a corrupt subtree, and any other -// error is left for the next step to retry. +// tried again, and an error that outlasts the attempts is left for the next step to retry. func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { var err error @@ -713,12 +712,9 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha } } - // Only a subtree that still cannot be deserialized is corrupt. Any other error, such - // as a blob store that is briefly down, is returned as it is, so the step is retried. - if errors.Is(err, errors.ErrProcessing) { - return nil, errors.NewSubtreeNotFoundError("failed to read subtree %s", subtreeHash.String(), err) - } - + // Not a permanent error, even when the subtree still cannot be deserialized: a short + // read looks the same as a corrupt file, and skipping the subtree would leave its txs + // looking mined for good. The step is retried; a job that keeps failing is set aside. return nil, err } diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 3b9384cf93..fe8130d7e8 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -722,7 +722,21 @@ func TestRepairOvertakenTips_RunsOnTheSetMinedWorker(t *testing.T) { }, 10*time.Second, 10*time.Millisecond, "the setMined worker marks B mined") require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: B's txs are mined") - _ = f.overtakeB(t) + // B' writes shared mined before C' overtakes B. The test writes it straight to the + // store, outside the worker, so a repair slice running in between would race it. + time.Sleep(10 * time.Millisecond) + + blockB2 := coinbaseOnlyBlock(t, f.tSettings.ChainCfgParams.GenesisHash, f.nBits, 1, 0xb2) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockB2, "test", blockchainoptions.WithSubtreesSet(true))) + require.True(t, f.onChain(t, f.blockBID), "precondition: B' alone does not overtake B") + + _, metaB2, err := f.blockchainClient.GetBlockHeader(f.ctx, blockB2.Hash()) + require.NoError(t, err) + f.mineSharedIn(t, metaB2.ID) + + blockC2 := coinbaseOnlyBlock(t, blockB2.Hash(), f.nBits, 2, 0xc2) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockC2, "test", blockchainoptions.WithSubtreesSet(true))) + require.False(t, f.onChain(t, f.blockBID), "precondition: the competing chain overtook B") require.Eventually(t, func() bool { m, err := f.store.Store.Get(f.ctx, f.bOnly, fields.UnminedSince) @@ -1196,3 +1210,26 @@ func TestRepairOvertakenTips_TransientSubtreeReadErrorRetried(t *testing.T) { require.NoError(t, err) require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once the store is back") } + +// TestRepairOvertakenTips_ShortReadsOutlastAttemptsRetried: short subtree reads through every +// read attempt of a step must not get the subtree skipped as corrupt, which would leave its +// txs looking mined for good; the step is retried and recovers once reads are whole again. +func TestRepairOvertakenTips_ShortReadsOutlastAttemptsRetried(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + store := &truncatingSubtreeStore{Store: f.subtreeStore} + store.truncate.Store(tipRepairSubtreeReadAttempts) + bv.subtreeStore = store + + _, err := drainTipRepair(t, bv) + require.Error(t, err, "the step fails while reads are short") + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is not skipped") + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once reads are whole") +} From 728aa735326aaa8dacf802b9d6674c007515e198 Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 13:48:30 +0100 Subject: [PATCH 09/33] fix(blockvalidation): skip a subtree that stays corrupt after max failed repair steps An unreadable subtree was retried forever: the job failed, was set aside, started again from subtree 0 and failed at the same subtree until the block left the window, so the subtrees after it were never repaired. A deserialize failure is now a SubtreeInvalid error; when it is still failing on the job's tipRepairMaxAttempts-th failed step in a row, the subtree is skipped with an error log and the rest of the block is repaired. A short run of short reads still cannot cause a skip. Updates the jobStep godoc. --- .../blockvalidation/overtaken_tip_repair.go | 18 ++++++++----- .../settxmined_overtaken_test.go | 25 +++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 257b356219..c295c78089 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -554,8 +554,10 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch } // jobStep writes the next slice of the job's txs, first loading the next subtree if the -// current one is used up, and finishes the job with its last slice. A subtree that is gone -// or unreadable is skipped with an error log, so the rest of the block is still repaired. +// current one is used up, and finishes the job with its last slice. A subtree that is gone, +// or still cannot be deserialized on the job's tipRepairMaxAttempts-th failed step in a row, +// is skipped with an error log, so the rest of the block is still repaired. Any other load +// error fails the step, to be retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { if job.nextSubtree >= len(job.subtrees) { @@ -567,7 +569,11 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha nodes, err := u.loadSubtreeNodes(ctx, subtreeHash) if err != nil { - if !isPermanentRepairError(err) { + // The failed steps in a row are all at this subtree: a step that finished the + // previous one succeeded and reset the count. A short read looks the same as a + // corrupt file, so only that many failures make it corrupt. + corrupt := errors.Is(err, errors.ErrSubtreeInvalid) && job.failures+1 >= tipRepairMaxAttempts + if !corrupt && !isPermanentRepairError(err) { return errors.NewProcessingError("[repairOvertakenTips][%s] failed to load subtree %d", job.hash.String(), job.nextSubtree, err) } @@ -713,8 +719,8 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha } // Not a permanent error, even when the subtree still cannot be deserialized: a short - // read looks the same as a corrupt file, and skipping the subtree would leave its txs - // looking mined for good. The step is retried; a job that keeps failing is set aside. + // read looks the same as a corrupt file. The step is retried, and jobStep skips the + // subtree only after it has failed for tipRepairMaxAttempts steps. return nil, err } @@ -734,7 +740,7 @@ func (u *BlockValidation) readSubtreeNodes(ctx context.Context, subtreeHash *cha subtree, err := subtreepkg.NewSubtreeFromReader(bufio.NewReader(reader)) if err != nil { - return nil, errors.NewProcessingError("failed to deserialize subtree %s", subtreeHash.String(), err) + return nil, errors.NewSubtreeInvalidError("failed to deserialize subtree %s", subtreeHash.String(), err) } return subtree.Nodes, nil diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index fe8130d7e8..1bae8fa416 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -1233,3 +1233,28 @@ func TestRepairOvertakenTips_ShortReadsOutlastAttemptsRetried(t *testing.T) { require.NoError(t, err) require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once reads are whole") } + +// TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts: a subtree that can never be +// deserialized must not hold the job forever, which would leave the block's later subtrees +// unrepaired. It is retried for tipRepairMaxAttempts steps, then skipped and the job goes on. +func TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + store := &truncatingSubtreeStore{Store: f.subtreeStore} + store.truncate.Store(1 << 30) + bv.subtreeStore = store + + for i := 1; i < tipRepairMaxAttempts; i++ { + _, err := drainTipRepair(t, bv) + require.Error(t, err, "step %d fails", i) + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is not skipped after %d steps", i) + } + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is skipped and the job finishes") +} From bbff7747cf07beafe7443b822ffe309b34613f91 Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 14:27:10 +0100 Subject: [PATCH 10/33] test(blockvalidation): a corrupt subtree does not block the overtaken block's later subtrees --- .../settxmined_overtaken_test.go | 89 ++++++++++++++++++- 1 file changed, 87 insertions(+), 2 deletions(-) diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 1bae8fa416..2f2b3ab767 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -1,6 +1,7 @@ package blockvalidation import ( + "bytes" "context" "io" "net/url" @@ -111,10 +112,25 @@ type overtakeFixture struct { bOnlyTx *bt.Tx bOnly *chainhash.Hash shared *chainhash.Hash + later *chainhash.Hash // only with newOvertakeFixtureWithLaterSubtree } func newOvertakeFixture(t *testing.T) *overtakeFixture { t.Helper() + + return newOvertakeFixtureSubtrees(t, false) +} + +// newOvertakeFixtureWithLaterSubtree is newOvertakeFixture with a second subtree in B that +// mines one more mempool tx, later, only ever in B. +func newOvertakeFixtureWithLaterSubtree(t *testing.T) *overtakeFixture { + t.Helper() + + return newOvertakeFixtureSubtrees(t, true) +} + +func newOvertakeFixtureSubtrees(t *testing.T, withLater bool) *overtakeFixture { + t.Helper() initPrometheusMetrics() ctx, cancel := context.WithCancel(context.Background()) @@ -149,6 +165,31 @@ func newOvertakeFixture(t *testing.T) *overtakeFixture { require.NoError(t, err) require.NoError(t, subtreeStore.Set(ctx, subtree.RootHash()[:], fileformat.FileTypeSubtree, subtreeBytes)) + subtreeHashes := []*chainhash.Hash{subtree.RootHash()} + txCount := uint64(subtree.Length()) //nolint:gosec + totalSize := uint64(coinbaseB.Size() + bOnlyTx.Size() + sharedTx.Size()) //nolint:gosec + + var later *chainhash.Hash + + if withLater { + _, laterTx, _, _ := createCoinbaseAndChildTx(t) + _, _, err = innerUtxoStore.SpendAndCreate(ctx, laterTx, 1, utxostore.WithCreateOnly()) + require.NoError(t, err) + + laterSubtree, err := subtreepkg.NewTreeByLeafCount(2) + require.NoError(t, err) + require.NoError(t, laterSubtree.AddNode(*laterTx.TxIDChainHash(), uint64(laterTx.Size()), 0)) //nolint:gosec + + laterBytes, err := laterSubtree.Serialize() + require.NoError(t, err) + require.NoError(t, subtreeStore.Set(ctx, laterSubtree.RootHash()[:], fileformat.FileTypeSubtree, laterBytes)) + + subtreeHashes = append(subtreeHashes, laterSubtree.RootHash()) + txCount += uint64(laterSubtree.Length()) //nolint:gosec + totalSize += uint64(laterTx.Size()) //nolint:gosec + later = laterTx.TxIDChainHash() + } + replicated := subtree.Duplicate() replicated.ReplaceRootNode(coinbaseB.TxIDChainHash(), 0, uint64(coinbaseB.Size())) //nolint:gosec @@ -156,8 +197,7 @@ func newOvertakeFixture(t *testing.T) *overtakeFixture { require.NoError(t, err) headerB := mineBlockHeader(t, tSettings.ChainCfgParams.GenesisHash, replicated.RootHash(), nBits) - totalSize := uint64(coinbaseB.Size() + bOnlyTx.Size() + sharedTx.Size()) //nolint:gosec - blockB, err := model.NewBlock(headerB, coinbaseB, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), totalSize, 1, 0) //nolint:gosec + blockB, err := model.NewBlock(headerB, coinbaseB, subtreeHashes, txCount, totalSize, 1, 0) require.NoError(t, err) require.NoError(t, blockchainClient.AddBlock(ctx, blockB, "test", blockchainoptions.WithSubtreesSet(true))) @@ -179,6 +219,7 @@ func newOvertakeFixture(t *testing.T) *overtakeFixture { bOnlyTx: bOnlyTx, bOnly: bOnlyTx.TxIDChainHash(), shared: sharedTx.TxIDChainHash(), + later: later, } } @@ -1258,3 +1299,47 @@ func TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts(t *testing.T) require.NoError(t, err) require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is skipped and the job finishes") } + +// corruptSubtreeStore returns a truncated reader for every read of one subtree: a subtree +// file that is corrupt on disk. +type corruptSubtreeStore struct { + blob.Store + key []byte +} + +func (s *corruptSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { + reader, err := s.Store.GetIoReader(ctx, key, fileType, opts...) + if err != nil || !bytes.Equal(key, s.key) { + return reader, err + } + + return struct { + io.Reader + io.Closer + }{io.LimitReader(reader, 10), reader}, nil +} + +// TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees: a corrupt earlier +// subtree of the overtaken block must not keep its later, intact subtrees from being +// repaired. Their txs would otherwise keep looking mined while off the chain. +func TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + require.Zero(t, f.unminedSince(t, f.later), "precondition: setTxMined marked the later subtree's tx mined") + _ = f.overtakeB(t) + + bv.subtreeStore = &corruptSubtreeStore{Store: f.subtreeStore, key: f.blockB.Subtrees[0][:]} + + for i := 1; i < tipRepairMaxAttempts; i++ { + _, err := drainTipRepair(t, bv) + require.Error(t, err, "step %d fails at the corrupt subtree", i) + require.Zero(t, f.unminedSince(t, f.later), "the corrupt subtree is retried, not skipped, after %d steps", i) + } + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.later), "the later, intact subtree is repaired") + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done) +} From 8c5f3632cc813362c424f480a9f2b1eea36afe2e Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 15:23:50 +0100 Subject: [PATCH 11/33] fix(blockvalidation): set an unreadable subtree aside for a later repair, count failures per subtree A subtree skipped after its tenth failed load was never read again, so a long run of short reads left its txs looking mined after storage recovered. The skip counted the job's failed steps, so a subtree right after one that was skipped inherited nine failures and was skipped after one more. Failed loads are now counted per subtree. A subtree that still cannot be deserialized after tipRepairMaxAttempts loads is set aside and the job goes on to the later subtrees; when it ends, the block is not done, holds the set-aside subtrees as pending and is deferred, and its next unmark job reads only those. A completed setTxMined write, or the block coming back onto the chain, clears pending so the whole block is repaired next time. --- .../blockvalidation/overtaken_tip_repair.go | 54 ++++++-- .../settxmined_overtaken_test.go | 127 ++++++++++++++++-- 2 files changed, 155 insertions(+), 26 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index c295c78089..fec4fc5339 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -106,6 +106,9 @@ type minedOnChainBlock struct { // deferred is set when the block's job kept failing and was set aside, so other // blocks are repaired first. deferred bool + // pending holds the subtrees an unmark job could not read and set aside; the block is + // not done, and its next unmark job reads only these. nil means all of them. + pending []*chainhash.Hash } // minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker @@ -168,6 +171,7 @@ func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { } b.done = false + b.pending = nil t.blocks[hash] = b return true @@ -253,6 +257,10 @@ type tipRepairJob struct { total int // failures counts the consecutive failed steps. failures int + // subtreeFailures counts the failed loads of the subtree at nextSubtree. + subtreeFailures int + // unreadable holds the subtrees set aside because they could not be read. + unreadable []*chainhash.Hash } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -402,6 +410,12 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b u.minedOnChain.drop(hash) delete(tracked, hash) + case onMainChain[b.id] && b.pending != nil: + // Back on the chain: if it leaves again, all of it needs repairing. + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.pending = nil }) + b.pending = nil + tracked[hash] = b + case onMainChain[b.id] && b.done && !b.stamped: // Back on the chain and nothing was re-marked: nothing to undo, but if it // leaves again it needs repairing again. @@ -545,19 +559,26 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch if kind == remineJob { e.done = false + e.pending = nil } }) - u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: block.Subtrees} + subtrees := block.Subtrees + if kind == unmarkJob && b.pending != nil { + subtrees = b.pending + } + + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees} return nil } // jobStep writes the next slice of the job's txs, first loading the next subtree if the -// current one is used up, and finishes the job with its last slice. A subtree that is gone, -// or still cannot be deserialized on the job's tipRepairMaxAttempts-th failed step in a row, -// is skipped with an error log, so the rest of the block is still repaired. Any other load -// error fails the step, to be retried. +// current one is used up, and finishes the job with its last slice. A subtree that is gone +// is skipped with an error log. One that still cannot be deserialized after +// tipRepairMaxAttempts failed loads is set aside, so the rest of the block is repaired, and +// read again by the block's next job: a run of short reads looks the same as a corrupt +// file. Any other load error fails the step, to be retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { if job.nextSubtree >= len(job.subtrees) { @@ -569,18 +590,21 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha nodes, err := u.loadSubtreeNodes(ctx, subtreeHash) if err != nil { - // The failed steps in a row are all at this subtree: a step that finished the - // previous one succeeded and reset the count. A short read looks the same as a - // corrupt file, so only that many failures make it corrupt. - corrupt := errors.Is(err, errors.ErrSubtreeInvalid) && job.failures+1 >= tipRepairMaxAttempts - if !corrupt && !isPermanentRepairError(err) { + job.subtreeFailures++ + + switch { + case isPermanentRepairError(err): + u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) + case errors.Is(err, errors.ErrSubtreeInvalid) && job.subtreeFailures >= tipRepairMaxAttempts: + u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) + job.unreadable = append(job.unreadable, subtreeHash) + default: return errors.NewProcessingError("[repairOvertakenTips][%s] failed to load subtree %d", job.hash.String(), job.nextSubtree, err) } - - u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) } job.nextSubtree++ + job.subtreeFailures = 0 job.current = nodes } @@ -649,8 +673,12 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { switch job.kind { case unmarkJob: + // A block with subtrees set aside is not done: its next job reads them again, + // after the other blocks have had their turn. u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { - e.done = true + e.done = len(job.unreadable) == 0 + e.pending = job.unreadable + e.deferred = !e.done e.failures = 0 }) diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 2f2b3ab767..68378d232c 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -1277,7 +1277,8 @@ func TestRepairOvertakenTips_ShortReadsOutlastAttemptsRetried(t *testing.T) { // TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts: a subtree that can never be // deserialized must not hold the job forever, which would leave the block's later subtrees -// unrepaired. It is retried for tipRepairMaxAttempts steps, then skipped and the job goes on. +// unrepaired. It is retried for tipRepairMaxAttempts steps, then set aside and the job +// finishes, with the block not done and the subtree pending for its next job. func TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) @@ -1295,21 +1296,27 @@ func TestRepairOvertakenTips_CorruptSubtreeSkippedAfterMaxAttempts(t *testing.T) require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is not skipped after %d steps", i) } - _, err := drainTipRepair(t, bv) + _, err := bv.tipRepairSlice(f.ctx, bv.blockchainClient) require.NoError(t, err) - require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the subtree is skipped and the job finishes") + require.Nil(t, bv.tipRepairJob, "the subtree is set aside and the job finishes") + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, b.done) + require.True(t, b.deferred, "other blocks go first") + require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending) } -// corruptSubtreeStore returns a truncated reader for every read of one subtree: a subtree -// file that is corrupt on disk. -type corruptSubtreeStore struct { +// shortReadSubtreeStore returns a truncated reader for the next short reads of one subtree; +// set short to 1<<30 for a subtree file that is corrupt on disk. +type shortReadSubtreeStore struct { blob.Store - key []byte + key []byte + short atomic.Int32 } -func (s *corruptSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { +func (s *shortReadSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { reader, err := s.Store.GetIoReader(ctx, key, fileType, opts...) - if err != nil || !bytes.Equal(key, s.key) { + if err != nil || !bytes.Equal(key, s.key) || s.short.Add(-1) < 0 { return reader, err } @@ -1319,9 +1326,27 @@ func (s *corruptSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileT }{io.LimitReader(reader, 10), reader}, nil } +// shortReads wraps store so the next n reads of the block's subtree idx are short. +func (f *overtakeFixture) shortReads(store blob.Store, idx int, n int32) *shortReadSubtreeStore { + s := &shortReadSubtreeStore{Store: store, key: f.blockB.Subtrees[idx][:]} + s.short.Store(n) + + return s +} + +// settleTipRepair runs the repair, as the worker's periodic tick would, until settled holds +// or a bound is hit, ignoring failed steps. +func settleTipRepair(t *testing.T, bv *BlockValidation, settled func() bool) { + t.Helper() + + for i := 0; i < 10*tipRepairMaxAttempts && !settled(); i++ { + _, _ = drainTipRepair(t, bv) + } +} + // TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees: a corrupt earlier // subtree of the overtaken block must not keep its later, intact subtrees from being -// repaired. Their txs would otherwise keep looking mined while off the chain. +// repaired, and must stay pending: the block is not done while it is unread. func TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees(t *testing.T) { f := newOvertakeFixtureWithLaterSubtree(t) bv := f.newBlockValidation(t) @@ -1330,7 +1355,7 @@ func TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees(t *testing. require.Zero(t, f.unminedSince(t, f.later), "precondition: setTxMined marked the later subtree's tx mined") _ = f.overtakeB(t) - bv.subtreeStore = &corruptSubtreeStore{Store: f.subtreeStore, key: f.blockB.Subtrees[0][:]} + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, 1<<30) for i := 1; i < tipRepairMaxAttempts; i++ { _, err := drainTipRepair(t, bv) @@ -1338,8 +1363,84 @@ func TestRepairOvertakenTips_CorruptSubtreeDoesNotBlockLaterSubtrees(t *testing. require.Zero(t, f.unminedSince(t, f.later), "the corrupt subtree is retried, not skipped, after %d steps", i) } - _, err := drainTipRepair(t, bv) - require.NoError(t, err) + settleTipRepair(t, bv, func() bool { return f.unminedSince(t, f.later) != 0 }) require.NotZero(t, f.unminedSince(t, f.later), "the later, intact subtree is repaired") + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, b.done, "the block is not done while a subtree is unread") + require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending, "the corrupt subtree is read again later") +} + +// TestRepairOvertakenTips_ShortReadsThenRecoveryRepairsSkippedSubtree: a subtree skipped +// after a long run of short reads must be repaired once its reads are whole again, not left +// with its txs looking mined. The later subtree is repaired meanwhile. +func TestRepairOvertakenTips_ShortReadsThenRecoveryRepairsSkippedSubtree(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, 3*tipRepairSubtreeReadAttempts*tipRepairMaxAttempts) + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].done }) + + require.NotZero(t, f.unminedSince(t, f.later), "the later subtree is repaired") + require.NotZero(t, f.unminedSince(t, f.bOnly), "the skipped subtree is repaired once its reads are whole") require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done) } + +// TestRepairOvertakenTips_SkipFailuresCountedPerSubtree: the failed steps that got a corrupt +// subtree skipped must not count toward skipping the next one. One step of short reads on +// the intact later subtree, right after the skip, must leave it to be retried. +func TestRepairOvertakenTips_SkipFailuresCountedPerSubtree(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + corrupt := f.shortReads(f.subtreeStore, 0, 1<<30) + bv.subtreeStore = f.shortReads(corrupt, 1, tipRepairSubtreeReadAttempts) + + settleTipRepair(t, bv, func() bool { return f.unminedSince(t, f.later) != 0 }) + + require.NotZero(t, f.unminedSince(t, f.later), "the later subtree is retried and repaired, not skipped") +} + +// TestRepairOvertakenTips_PendingClearedBackOnChain: a block with a subtree set aside that +// comes back onto the chain must be repaired in full if it leaves again: its other txs may +// have been marked mined again meanwhile. +func TestRepairOvertakenTips_PendingClearedBackOnChain(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, 1<<30) + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + require.NotNil(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending, "precondition: a subtree is set aside") + + f.extendB(t) + + _, _ = bv.tipRepairSlice(f.ctx, bv.blockchainClient) + require.Nil(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending) +} + +// TestMinedOnChainTracker_WriteClearsPending: a completed setTxMined write may have cleared +// any of the block's txs, so the next repair reads the whole block, not only the subtrees +// set aside. +func TestMinedOnChainTracker_WriteClearsPending(t *testing.T) { + tracker := newMinedOnChainTracker() + hash := chainhash.HashH([]byte("b")) + subtree := chainhash.HashH([]byte("s")) + + tracker.track(hash, 1, 1) + + gen := tracker.snapshot()[hash].gen + tracker.update(hash, gen, func(e *minedOnChainBlock) { e.pending = []*chainhash.Hash{&subtree} }) + + require.True(t, tracker.markWritten(hash)) + require.Nil(t, tracker.snapshot()[hash].pending) +} From f7899cb46caae1d323f7d554aff3e4fe6f89d498 Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 16:35:50 +0100 Subject: [PATCH 12/33] test(blockvalidation): retry a SetMinedMulti the sqlitememory engine rejects as a table-lock deadlock --- services/blockvalidation/settxmined_overtaken_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 68378d232c..ed104d946e 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -6,6 +6,7 @@ import ( "io" "net/url" "slices" + "strings" "sync" "sync/atomic" "testing" @@ -51,7 +52,14 @@ type overtakeTestStore struct { } func (s *overtakeTestStore) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { + // The sqlitememory engine reports two of its connections waiting on each other as a + // table lock, which the sql store's SetMinedMulti does not retry; SQLite's remedy is to + // run the transaction again. res, err := s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) + for attempt := 1; attempt < 5 && err != nil && strings.Contains(err.Error(), "database table is locked"); attempt++ { + res, err = s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) + } + if err == nil && s.failAfterSetMined.Load() { return nil, errors.NewStorageError("injected failure after the write") } From 928aadf7a6efb88262b2d9b111d2db489e60ba56 Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 16:50:44 +0100 Subject: [PATCH 13/33] fix(utxo/sql): retry a SetMinedMulti chunk the database refuses with a lock error Under SQLite's shared cache a lock cycle between connections fails one side with SQLITE_LOCKED. When that side was a SetMinedMulti chunk, the error was returned and setTxMined failed the whole block, though the spend batch, the rollback and the parent-output read all retry the same error. Each chunk is its own transaction and is rolled back on failure, so it is now tried up to three times while isDeadlock matches, with a short backoff. Drops the retry the blockvalidation repair tests' store wrapper did for this. --- .../settxmined_overtaken_test.go | 8 -- stores/utxo/sql/setmined_lock_retry_test.go | 90 +++++++++++++++++++ stores/utxo/sql/sql.go | 34 ++++++- 3 files changed, 123 insertions(+), 9 deletions(-) create mode 100644 stores/utxo/sql/setmined_lock_retry_test.go diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index ed104d946e..68378d232c 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -6,7 +6,6 @@ import ( "io" "net/url" "slices" - "strings" "sync" "sync/atomic" "testing" @@ -52,14 +51,7 @@ type overtakeTestStore struct { } func (s *overtakeTestStore) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxostore.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { - // The sqlitememory engine reports two of its connections waiting on each other as a - // table lock, which the sql store's SetMinedMulti does not retry; SQLite's remedy is to - // run the transaction again. res, err := s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) - for attempt := 1; attempt < 5 && err != nil && strings.Contains(err.Error(), "database table is locked"); attempt++ { - res, err = s.Store.SetMinedMulti(ctx, hashes, minedBlockInfo) - } - if err == nil && s.failAfterSetMined.Load() { return nil, errors.NewStorageError("injected failure after the write") } diff --git a/stores/utxo/sql/setmined_lock_retry_test.go b/stores/utxo/sql/setmined_lock_retry_test.go new file mode 100644 index 0000000000..901a819f14 --- /dev/null +++ b/stores/utxo/sql/setmined_lock_retry_test.go @@ -0,0 +1,90 @@ +package sql + +import ( + "context" + "testing" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/stores/utxo" + "github.com/bsv-blockchain/teranode/ulogger" + "github.com/stretchr/testify/require" +) + +// A lock error on a SetMinedMulti chunk is retried, not returned. Under SQLite's +// shared cache a lock cycle between connections fails one side with +// SQLITE_LOCKED; when the failed side is SetMinedMulti, setTxMined failed the +// whole block over it. Each chunk is its own transaction and is rolled back on +// failure, so running it again is safe. The write is the real chunk write; the +// first attempts are refused with the error text the engine raised in +// services/blockvalidation's overtaken-tip repair tests. +func TestSetMinedMultiRetriesLockError(t *testing.T) { + ctx := context.Background() + db, tx := setup(ctx, t) + + _, err := db.Create(ctx, tx, 0) + require.NoError(t, err) + + hashes := []*chainhash.Hash{tx.TxIDChainHash()} + info := utxo.MinedBlockInfo{BlockID: 7, BlockHeight: 1, OnLongestChain: true} + lockErr := errors.NewStorageError("database table is locked: database is deadlocked (6)") + + t.Run("a refused write succeeds on a later attempt", func(t *testing.T) { + attempts := 0 + + var blockIDs map[chainhash.Hash][]uint32 + + err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + attempts++ + if attempts < setMinedMultiRetries { + return lockErr + } + + var err error + + blockIDs, err = db.setMinedMultiChunk(ctx, hashes, info) + + return err + }) + require.NoError(t, err) + require.Equal(t, setMinedMultiRetries, attempts) + require.Equal(t, []uint32{7}, blockIDs[*hashes[0]], "the tx is mined in the block once the lock clears") + }) + + t.Run("a lock that never clears returns the lock error", func(t *testing.T) { + attempts := 0 + + err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + attempts++ + return lockErr + }) + require.ErrorIs(t, err, lockErr) + require.Equal(t, setMinedMultiRetries, attempts) + }) + + t.Run("any other error is not retried", func(t *testing.T) { + attempts := 0 + other := errors.NewTxNotFoundError("transaction not found") + + err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + attempts++ + return other + }) + require.ErrorIs(t, err, other) + require.Equal(t, 1, attempts) + }) + + t.Run("a cancelled context stops the retries", func(t *testing.T) { + cctx, cancel := context.WithCancel(ctx) + attempts := 0 + + err := retryWriteOnLockError(cctx, ulogger.TestLogger{}, len(hashes), func() error { + attempts++ + cancel() + + return lockErr + }) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, attempts) + }) +} diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 01c346bf64..5c0101a9f0 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -3631,8 +3631,15 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min default: } - chunkResult, err := s.setMinedMultiChunk(ctx, hashes[i:end], minedBlockInfo) + var chunkResult map[chainhash.Hash][]uint32 + err := retryWriteOnLockError(ctx, s.logger, end-i, func() error { + var err error + + chunkResult, err = s.setMinedMultiChunk(ctx, hashes[i:end], minedBlockInfo) + + return err + }) if err != nil { return nil, errors.NewStorageError("SQL error in SetMinedMulti (chunk %d-%d): %v", i, end-1, err) } @@ -3645,6 +3652,31 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min return resultMap, nil } +// setMinedMultiRetries bounds how many times one SetMinedMulti chunk is tried +// when the database refuses it with a lock error. +const setMinedMultiRetries = 3 + +// retryWriteOnLockError runs write up to setMinedMultiRetries times while it +// fails with an error isDeadlock recognises, and returns its last error. A +// chunk is one transaction, rolled back when it fails, so running it again is +// safe. +func retryWriteOnLockError(ctx context.Context, logger ulogger.Logger, txs int, write func() error) error { + for attempt := 1; ; attempt++ { + err := write() + if err == nil || !isDeadlock(err) || attempt >= setMinedMultiRetries { + return err + } + + logger.Warnf("[SetMinedMulti] lock error (attempt %d/%d), retrying chunk of %d txs: %v", attempt, setMinedMultiRetries, txs, err) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt) * 10 * time.Millisecond): + } + } +} + // setMinedMultiChunk processes a single chunk of hashes (up to maxINClauseSize). // Uses portable IN ($1,$2,...,$N) clauses that work on both PostgreSQL and SQLite. func (s *Store) setMinedMultiChunk(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxo.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { From 6bb7935d17909b3e864e592c35b5eb80f9c1c869 Mon Sep 17 00:00:00 2001 From: Siggi Date: Wed, 28 Oct 2026 17:13:09 +0100 Subject: [PATCH 14/33] fix(blockvalidation): back off by best height between passes over set-aside subtrees A block whose subtree stays unreadable was read again as soon as no other block needed the repair, so a corrupt subtree took the repair slot and logged on every pass until the block left the window. The next pass now waits until the best height has moved on 2, 4, 8, ... up to 128 blocks from the start of the last one, about a dozen passes over the window; it still never gives the subtree up. Clearing pending resets the count. Also corrects the jobStep and loadSubtreeNodes comments: only an unmark job comes back to a set-aside subtree, and the remine job leaving one is the safe direction. The recovery test now uses the 30 short reads its description names. --- .../blockvalidation/overtaken_tip_repair.go | 61 +++++++++++++----- .../settxmined_overtaken_test.go | 62 +++++++++++++++++-- 2 files changed, 101 insertions(+), 22 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index fec4fc5339..55cfdf07a9 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -69,6 +69,10 @@ const ( // defaultTipRepairSliceSize is used when BlockValidation.TipRepairSliceSize is unset. defaultTipRepairSliceSize = 250_000 + // tipRepairPendingMaxBackoff caps, in blocks, how long a block with subtrees set aside + // waits for its next pass over them. + tipRepairPendingMaxBackoff = 128 + // tipRepairSubtreeReadAttempts and tipRepairSubtreeReadBackoff match Block.GetSubtrees. tipRepairSubtreeReadAttempts = 3 tipRepairSubtreeReadBackoff = 100 * time.Millisecond @@ -109,6 +113,18 @@ type minedOnChainBlock struct { // pending holds the subtrees an unmark job could not read and set aside; the block is // not done, and its next unmark job reads only these. nil means all of them. pending []*chainhash.Hash + // passes counts the unmark jobs that ended with subtrees set aside, and nextPassHeight + // is the best height the next one waits for, so a subtree that stays unreadable is read + // again a few times over the window rather than on every slice. + passes int + nextPassHeight uint32 +} + +// clearPending has the block's next unmark job read all of it again. +func (b *minedOnChainBlock) clearPending() { + b.pending = nil + b.passes = 0 + b.nextPassHeight = 0 } // minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker @@ -171,7 +187,7 @@ func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { } b.done = false - b.pending = nil + b.clearPending() t.blocks[hash] = b return true @@ -261,6 +277,8 @@ type tipRepairJob struct { subtreeFailures int // unreadable holds the subtrees set aside because they could not be read. unreadable []*chainhash.Hash + // startHeight is the best height when the job started. + startHeight uint32 } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -412,8 +430,8 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b case onMainChain[b.id] && b.pending != nil: // Back on the chain: if it leaves again, all of it needs repairing. - u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.pending = nil }) - b.pending = nil + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.clearPending() }) + b.clearPending() tracked[hash] = b case onMainChain[b.id] && b.done && !b.stamped: @@ -455,7 +473,7 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // Blocks set aside after failing come last. for _, deferred := range []bool{false, true} { - if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, deferred); started { + if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, bestHeight, deferred); started { return more, err } } @@ -466,16 +484,16 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // startNextJob starts a job on the next block that needs one, among the blocks whose deferred // flag matches, and reports whether it found one. func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blockchain.ClientI, tracked map[chainhash.Hash]minedOnChainBlock, - onMainChain map[uint32]bool, deferred bool) (more, started bool, err error) { + onMainChain map[uint32]bool, bestHeight uint32, deferred bool) (more, started bool, err error) { // A re-marked block back on the chain first: its txs look unmined while mined. for hash, b := range tracked { if b.deferred == deferred && b.stamped && onMainChain[b.id] { - return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b) + return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b, bestHeight) } } for hash, b := range tracked { - if b.deferred != deferred || b.done || onMainChain[b.id] { + if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && bestHeight < b.nextPassHeight) { continue } @@ -494,7 +512,7 @@ func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blo continue } - return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) + return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b, bestHeight) } return false, false, nil @@ -548,7 +566,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie } // startJob begins a job over the block: it reads the block's subtree list, not its txs. -func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock) error { +func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock, bestHeight uint32) error { block, err := blockchainClient.GetBlock(ctx, &hash) if err != nil { return u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block", hash.String(), err)) @@ -559,7 +577,7 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch if kind == remineJob { e.done = false - e.pending = nil + e.clearPending() } }) @@ -568,7 +586,7 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch subtrees = b.pending } - u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees} + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees, startHeight: bestHeight} return nil } @@ -576,9 +594,12 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone // is skipped with an error log. One that still cannot be deserialized after -// tipRepairMaxAttempts failed loads is set aside, so the rest of the block is repaired, and -// read again by the block's next job: a run of short reads looks the same as a corrupt -// file. Any other load error fails the step, to be retried. +// tipRepairMaxAttempts failed loads is set aside, so the rest of the block is repaired: a +// run of short reads looks the same as a corrupt file. An unmark job's set-aside subtrees +// are read again by the block's next unmark job (see finishJob). A remine job does not +// come back to them, which is the safe direction: their txs stay flagged unmined until +// fixUnminedSinceInconsistencies clears them. Any other load error fails the step, to be +// retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { if job.nextSubtree >= len(job.subtrees) { @@ -674,12 +695,18 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { switch job.kind { case unmarkJob: // A block with subtrees set aside is not done: its next job reads them again, - // after the other blocks have had their turn. + // after the other blocks have had their turn and the best height has moved on by + // twice as many blocks as last time. u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.done = len(job.unreadable) == 0 e.pending = job.unreadable e.deferred = !e.done e.failures = 0 + + if !e.done { + e.passes++ + e.nextPassHeight = job.startHeight + uint32(min(1< Date: Thu, 29 Oct 2026 09:49:23 +0100 Subject: [PATCH 15/33] fix(utxo/sql): recognise a wrapped PostgreSQL deadlock, test the SetMinedMulti retry through the store A teranode error keeps only the text of the driver error it wraps, so a 40P01 deadlock wrapped inside setMinedMultiChunk reached isDeadlock without its *pgconn.PgError and was not retried. isDeadlock now also matches the SQLSTATE in the text. The new store test refuses the chunk's first block_ids insert through a trigger with the shared-cache lock error, and fails when the retry at the SetMinedMulti call site is removed. --- stores/utxo/sql/setmined_lock_retry_test.go | 61 +++++++++++++++++++++ stores/utxo/sql/sql.go | 5 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/stores/utxo/sql/setmined_lock_retry_test.go b/stores/utxo/sql/setmined_lock_retry_test.go index 901a819f14..8f1cc05051 100644 --- a/stores/utxo/sql/setmined_lock_retry_test.go +++ b/stores/utxo/sql/setmined_lock_retry_test.go @@ -2,13 +2,17 @@ package sql import ( "context" + "database/sql/driver" + "sync/atomic" "testing" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/ulogger" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" + "modernc.org/sqlite" ) // A lock error on a SetMinedMulti chunk is retried, not returned. Under SQLite's @@ -88,3 +92,60 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { require.Equal(t, 1, attempts) }) } + +// setMinedLockRefusals is how many more block_ids inserts the +// setmined_lock_refusal trigger refuses with SQLite's shared-cache lock error. +var setMinedLockRefusals atomic.Int32 + +// Registered in init so every connection the sqlite driver opens has it. +func init() { + sqlite.MustRegisterScalarFunction("setmined_lock_refusal", 0, func(*sqlite.FunctionContext, []driver.Value) (driver.Value, error) { + if setMinedLockRefusals.Add(-1) >= 0 { + return nil, errors.NewStorageError("database table is locked: database is deadlocked (6)") + } + + return int64(0), nil + }) +} + +// SetMinedMulti itself retries a chunk the engine refuses with a lock error. A +// trigger refuses the chunk's first block_ids insert with the error SQLite's +// shared cache raises for a lock cycle; the chunk rolls back and the retried +// attempt mines the tx. Without the retry at the SetMinedMulti call site the +// call returns the lock error. +func TestSetMinedMultiRetriesLockErrorFromChunk(t *testing.T) { + ctx := context.Background() + db, tx := setup(ctx, t) + + _, err := db.Create(ctx, tx, 0) + require.NoError(t, err) + + _, err = db.db.ExecContext(ctx, `CREATE TRIGGER setmined_lock_refusal BEFORE INSERT ON block_ids + BEGIN SELECT setmined_lock_refusal(); END`) + require.NoError(t, err) + + t.Cleanup(func() { + setMinedLockRefusals.Store(0) + _, _ = db.db.ExecContext(ctx, "DROP TRIGGER IF EXISTS setmined_lock_refusal") + }) + + setMinedLockRefusals.Store(1) + + hash := tx.TxIDChainHash() + + blockIDs, err := db.SetMinedMulti(ctx, []*chainhash.Hash{hash}, utxo.MinedBlockInfo{BlockID: 7, BlockHeight: 1, OnLongestChain: true}) + require.NoError(t, err) + require.Equal(t, []uint32{7}, blockIDs[*hash]) + require.Equal(t, int32(-1), setMinedLockRefusals.Load(), "the first insert was refused and the second ran") +} + +// A PostgreSQL deadlock (SQLSTATE 40P01) is still recognised after +// setMinedMultiChunk wraps it in a teranode error, which keeps only the +// driver error's text. +func TestIsDeadlockWrappedPostgresError(t *testing.T) { + pgErr := &pgconn.PgError{Severity: "ERROR", Code: "40P01", Message: "deadlock detected"} + + require.True(t, isDeadlock(pgErr)) + require.True(t, isDeadlock(errors.NewStorageError("SQL error inserting block_ids: %v", pgErr))) + require.False(t, isDeadlock(errors.NewStorageError("SQL error inserting block_ids: %v", &pgconn.PgError{Severity: "ERROR", Code: "23505", Message: "duplicate key"}))) +} diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 5c0101a9f0..62e0cd84ff 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -2414,8 +2414,11 @@ func isDeadlock(err error) bool { if errors.As(err, &sqliteErr) && usql.IsSQLiteLockCode(sqliteErr.Code()) { return true } + // A teranode error keeps only the text of a driver error it wraps, so a + // PostgreSQL deadlock wrapped before it gets here is matched by its SQLSTATE. msg := err.Error() - return strings.Contains(msg, "database is locked") || strings.Contains(msg, "database table is locked") + return strings.Contains(msg, "database is locked") || strings.Contains(msg, "database table is locked") || + strings.Contains(msg, "(SQLSTATE "+usql.PgErrDeadlockDetected+")") } // sendSpendBatch is the batcher callback that processes a batch of spend operations From a3b25e93add5d1767d787d56e36b8a33ef843752 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 09:58:07 +0100 Subject: [PATCH 16/33] fix(blockvalidation): back off by time, not best height, between passes over set-aside subtrees A height back-off of up to 128 blocks let a subtree that recovered late in the window wait past the default 288-block retention, so a parent its off-chain txs spent could be pruned before the repair cleared them. The next pass now waits 1s, doubling up to a minute, on the tracker's clock: a subtree that recovers is repaired within a minute, and one that stays corrupt is read at most once a minute. --- .../blockvalidation/overtaken_tip_repair.go | 46 +++++------ .../settxmined_overtaken_test.go | 77 ++++++++++++++----- 2 files changed, 80 insertions(+), 43 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 55cfdf07a9..22a5ed176b 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -69,9 +69,12 @@ const ( // defaultTipRepairSliceSize is used when BlockValidation.TipRepairSliceSize is unset. defaultTipRepairSliceSize = 250_000 - // tipRepairPendingMaxBackoff caps, in blocks, how long a block with subtrees set aside - // waits for its next pass over them. - tipRepairPendingMaxBackoff = 128 + // tipRepairPendingBaseBackoff is how long a block with subtrees set aside waits for its + // first pass over them again; each later pass waits twice as long, up to + // tipRepairPendingMaxBackoff. The wait is in time, not blocks, so a subtree that recovers + // is repaired well before the pruner can delete a parent its off-chain txs spent. + tipRepairPendingBaseBackoff = time.Second + tipRepairPendingMaxBackoff = time.Minute // tipRepairSubtreeReadAttempts and tipRepairSubtreeReadBackoff match Block.GetSubtrees. tipRepairSubtreeReadAttempts = 3 @@ -113,18 +116,18 @@ type minedOnChainBlock struct { // pending holds the subtrees an unmark job could not read and set aside; the block is // not done, and its next unmark job reads only these. nil means all of them. pending []*chainhash.Hash - // passes counts the unmark jobs that ended with subtrees set aside, and nextPassHeight - // is the best height the next one waits for, so a subtree that stays unreadable is read - // again a few times over the window rather than on every slice. - passes int - nextPassHeight uint32 + // passes counts the unmark jobs that ended with subtrees set aside, and nextPass is + // when the next one may start, so a subtree that stays unreadable is not read again on + // every slice. + passes int + nextPass time.Time } // clearPending has the block's next unmark job read all of it again. func (b *minedOnChainBlock) clearPending() { b.pending = nil b.passes = 0 - b.nextPassHeight = 0 + b.nextPass = time.Time{} } // minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker @@ -133,10 +136,12 @@ type minedOnChainTracker struct { mu sync.Mutex blocks map[chainhash.Hash]minedOnChainBlock nextGen uint64 + // now times the back-off between passes over subtrees set aside. + now func() time.Time } func newMinedOnChainTracker() *minedOnChainTracker { - return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock)} + return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock), now: time.Now} } // track watches a block whose on-chain write is starting: the write clears the block's txs @@ -277,8 +282,6 @@ type tipRepairJob struct { subtreeFailures int // unreadable holds the subtrees set aside because they could not be read. unreadable []*chainhash.Hash - // startHeight is the best height when the job started. - startHeight uint32 } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -473,7 +476,7 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // Blocks set aside after failing come last. for _, deferred := range []bool{false, true} { - if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, bestHeight, deferred); started { + if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, deferred); started { return more, err } } @@ -484,16 +487,16 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // startNextJob starts a job on the next block that needs one, among the blocks whose deferred // flag matches, and reports whether it found one. func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blockchain.ClientI, tracked map[chainhash.Hash]minedOnChainBlock, - onMainChain map[uint32]bool, bestHeight uint32, deferred bool) (more, started bool, err error) { + onMainChain map[uint32]bool, deferred bool) (more, started bool, err error) { // A re-marked block back on the chain first: its txs look unmined while mined. for hash, b := range tracked { if b.deferred == deferred && b.stamped && onMainChain[b.id] { - return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b, bestHeight) + return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b) } } for hash, b := range tracked { - if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && bestHeight < b.nextPassHeight) { + if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && u.minedOnChain.now().Before(b.nextPass)) { continue } @@ -512,7 +515,7 @@ func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blo continue } - return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b, bestHeight) + return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) } return false, false, nil @@ -566,7 +569,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie } // startJob begins a job over the block: it reads the block's subtree list, not its txs. -func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock, bestHeight uint32) error { +func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock) error { block, err := blockchainClient.GetBlock(ctx, &hash) if err != nil { return u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block", hash.String(), err)) @@ -586,7 +589,7 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch subtrees = b.pending } - u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees, startHeight: bestHeight} + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees} return nil } @@ -695,8 +698,7 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { switch job.kind { case unmarkJob: // A block with subtrees set aside is not done: its next job reads them again, - // after the other blocks have had their turn and the best height has moved on by - // twice as many blocks as last time. + // after the other blocks have had their turn and twice as long as last time. u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.done = len(job.unreadable) == 0 e.pending = job.unreadable @@ -705,7 +707,7 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { if !e.done { e.passes++ - e.nextPassHeight = job.startHeight + uint32(min(1< Date: Thu, 29 Oct 2026 10:31:13 +0100 Subject: [PATCH 17/33] fix(blockvalidation): pass over set-aside subtrees on every new block, wake the idle worker when the back-off ends A pass waiting on the time back-off skipped a block that arrived meanwhile, and the pruner works block by block, so a parent the set-aside txs spent could reach its prune height while the recovered subtree waited. A pass now also comes due once the best height has moved past the one its last pass started at. With nothing else due, a slice cleared tipRepairRetry and nothing requested the pass when its back-off ended; on an idle chain it waited for the next block or write. The setMined worker now sets a timer for the earliest pending pass. --- services/blockvalidation/BlockValidation.go | 3 + .../blockvalidation/overtaken_tip_repair.go | 63 +++++++++++++---- .../settxmined_overtaken_test.go | 70 +++++++++++++++++++ 3 files changed, 123 insertions(+), 13 deletions(-) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 7997a10a5b..67f929041b 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -334,6 +334,9 @@ type BlockValidation struct { tipRepairBestHash chainhash.Hash // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. tipRepairRetry atomic.Bool + // tipRepairWake requests a slice when the earliest pass over subtrees set aside comes + // due. Only the setMined worker sets it. + tipRepairWake *time.Timer // revalidateBlockChan receives blocks that need revalidation revalidateBlockChan chan revalidateBlockData diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 22a5ed176b..7cf858ff9f 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -116,11 +116,12 @@ type minedOnChainBlock struct { // pending holds the subtrees an unmark job could not read and set aside; the block is // not done, and its next unmark job reads only these. nil means all of them. pending []*chainhash.Hash - // passes counts the unmark jobs that ended with subtrees set aside, and nextPass is - // when the next one may start, so a subtree that stays unreadable is not read again on - // every slice. - passes int - nextPass time.Time + // passes counts the unmark jobs that ended with subtrees set aside. The next one may + // start at nextPass, or sooner once the best height is above passHeight, so a subtree + // that stays unreadable is not read again on every slice, but is on every block. + passes int + nextPass time.Time + passHeight uint32 } // clearPending has the block's next unmark job read all of it again. @@ -128,6 +129,7 @@ func (b *minedOnChainBlock) clearPending() { b.pending = nil b.passes = 0 b.nextPass = time.Time{} + b.passHeight = 0 } // minedOnChainTracker holds the blocks the overtaken-tip repair watches. A nil tracker @@ -144,6 +146,26 @@ func newMinedOnChainTracker() *minedOnChainTracker { return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock), now: time.Now} } +// nextPendingPass returns the earliest nextPass of the blocks with subtrees set aside. +func (t *minedOnChainTracker) nextPendingPass() (time.Time, bool) { + if t == nil { + return time.Time{}, false + } + + t.mu.Lock() + defer t.mu.Unlock() + + var earliest time.Time + + for _, b := range t.blocks { + if b.pending != nil && !b.done && (earliest.IsZero() || b.nextPass.Before(earliest)) { + earliest = b.nextPass + } + } + + return earliest, !earliest.IsZero() +} + // track watches a block whose on-chain write is starting: the write clears the block's txs // again, so the block gets a new gen and starts over. func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32) { @@ -282,6 +304,8 @@ type tipRepairJob struct { subtreeFailures int // unreadable holds the subtrees set aside because they could not be read. unreadable []*chainhash.Hash + // startHeight is the best height when the job started. + startHeight uint32 } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -315,6 +339,17 @@ func (u *BlockValidation) runTipRepairSlice(ctx context.Context, blockchainClien if more { u.signalTipRepair() + return + } + + // Nothing is due now. A pass over subtrees set aside comes due at its nextPass even on + // an idle chain, with no block or write to request a slice, so wake for the earliest. + if at, ok := u.minedOnChain.nextPendingPass(); ok { + if u.tipRepairWake != nil { + u.tipRepairWake.Stop() + } + + u.tipRepairWake = time.AfterFunc(at.Sub(u.minedOnChain.now()), u.signalTipRepair) } } @@ -476,7 +511,7 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // Blocks set aside after failing come last. for _, deferred := range []bool{false, true} { - if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, deferred); started { + if more, started, err := u.startNextJob(ctx, blockchainClient, tracked, onMainChain, bestHeight, deferred); started { return more, err } } @@ -487,16 +522,16 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b // startNextJob starts a job on the next block that needs one, among the blocks whose deferred // flag matches, and reports whether it found one. func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blockchain.ClientI, tracked map[chainhash.Hash]minedOnChainBlock, - onMainChain map[uint32]bool, deferred bool) (more, started bool, err error) { + onMainChain map[uint32]bool, bestHeight uint32, deferred bool) (more, started bool, err error) { // A re-marked block back on the chain first: its txs look unmined while mined. for hash, b := range tracked { if b.deferred == deferred && b.stamped && onMainChain[b.id] { - return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b) + return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b, bestHeight) } } for hash, b := range tracked { - if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && u.minedOnChain.now().Before(b.nextPass)) { + if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && u.minedOnChain.now().Before(b.nextPass) && bestHeight <= b.passHeight) { continue } @@ -515,7 +550,7 @@ func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blo continue } - return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b) + return true, true, u.startJob(ctx, blockchainClient, unmarkJob, hash, b, bestHeight) } return false, false, nil @@ -569,7 +604,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie } // startJob begins a job over the block: it reads the block's subtree list, not its txs. -func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock) error { +func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockchain.ClientI, kind tipRepairJobKind, hash chainhash.Hash, b minedOnChainBlock, bestHeight uint32) error { block, err := blockchainClient.GetBlock(ctx, &hash) if err != nil { return u.repairFailed(hash, b.gen, errors.NewServiceError("[repairOvertakenTips][%s] failed to get block", hash.String(), err)) @@ -589,7 +624,7 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch subtrees = b.pending } - u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees} + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees, startHeight: bestHeight} return nil } @@ -698,7 +733,8 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { switch job.kind { case unmarkJob: // A block with subtrees set aside is not done: its next job reads them again, - // after the other blocks have had their turn and twice as long as last time. + // after the other blocks have had their turn, and once twice as long as last time + // has passed or a new block has arrived, whichever comes first. u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.done = len(job.unreadable) == 0 e.pending = job.unreadable @@ -708,6 +744,7 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { if !e.done { e.passes++ e.nextPass = u.minedOnChain.now().Add(min(tipRepairPendingBaseBackoff< Date: Thu, 29 Oct 2026 10:56:17 +0100 Subject: [PATCH 18/33] fix(blockvalidation): never treat a storage error as a missing subtree, set aside a subtree any persistent error keeps from loading The repair classed any error chain holding a not-found as permanent, so a storage error over a not-found skipped the subtree for good; it now follows the subtreeAbsent rule. A subtree that failed to load for a reason other than a deserialize failure failed every step until the job was deferred, which then started over from the first subtree; now any load error that lasts tipRepairMaxAttempts loads sets the subtree aside, to be read again later, and the rest of the block is repaired. --- .../blockvalidation/overtaken_tip_repair.go | 22 ++++--- .../settxmined_overtaken_test.go | 66 +++++++++++++++++++ 2 files changed, 78 insertions(+), 10 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 7cf858ff9f..a22545bac5 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -89,10 +89,11 @@ func tipRepairExpired(height, bestHeight uint32) bool { } // isPermanentRepairError reports whether a repair error means the block can never be -// repaired: the block or its subtrees are gone. +// repaired: the block or its subtrees are gone. A storage error is the store failing while +// it looked, whatever it wraps, so it is never permanent (the subtreeAbsent rule). func isPermanentRepairError(err error) bool { - return errors.Is(err, errors.ErrBlockNotFound) || errors.Is(err, errors.ErrSubtreeNotFound) || - errors.Is(err, errors.ErrBlobNotFound) || errors.Is(err, errors.ErrNotFound) + return (errors.Is(err, errors.ErrBlockNotFound) || errors.Is(err, errors.ErrSubtreeNotFound) || + errors.Is(err, errors.ErrBlobNotFound) || errors.Is(err, errors.ErrNotFound)) && !errors.Is(err, errors.ErrStorageError) } // minedOnChainBlock is a block whose txs setTxMined may have marked mined on the longest @@ -631,12 +632,13 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone -// is skipped with an error log. One that still cannot be deserialized after -// tipRepairMaxAttempts failed loads is set aside, so the rest of the block is repaired: a -// run of short reads looks the same as a corrupt file. An unmark job's set-aside subtrees +// is skipped with an error log. One that still cannot be loaded after tipRepairMaxAttempts +// failed loads, whatever the error, is set aside, so the rest of the block is repaired: a +// run of short reads looks the same as a corrupt file, and a store that stays down must not +// hold up the later subtrees. An unmark job's set-aside subtrees // are read again by the block's next unmark job (see finishJob). A remine job does not // come back to them, which is the safe direction: their txs stay flagged unmined until -// fixUnminedSinceInconsistencies clears them. Any other load error fails the step, to be +// fixUnminedSinceInconsistencies clears them. Until then a load error fails the step, to be // retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { @@ -654,7 +656,7 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha switch { case isPermanentRepairError(err): u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) - case errors.Is(err, errors.ErrSubtreeInvalid) && job.subtreeFailures >= tipRepairMaxAttempts: + case ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) job.unreadable = append(job.unreadable, subtreeHash) default: @@ -807,7 +809,7 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha return nodes, nil } - if errors.Is(err, errors.ErrNotFound) { + if subtreeAbsent(err) { return nil, err } } @@ -820,7 +822,7 @@ func (u *BlockValidation) loadSubtreeNodes(ctx context.Context, subtreeHash *cha func (u *BlockValidation) readSubtreeNodes(ctx context.Context, subtreeHash *chainhash.Hash) ([]subtreepkg.Node, error) { reader, err := u.subtreeStore.GetIoReader(ctx, subtreeHash[:], fileformat.FileTypeSubtree) - if errors.Is(err, errors.ErrNotFound) { + if subtreeAbsent(err) { reader, err = u.subtreeStore.GetIoReader(ctx, subtreeHash[:], fileformat.FileTypeSubtreeToCheck) } diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index b21f0e516a..dc24fd7289 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -1601,3 +1601,69 @@ func TestRepairOvertakenTips_PendingPassWakesIdleWorker(t *testing.T) { require.Eventually(t, func() bool { return f.unminedSince(t, f.bOnly) != 0 }, 10*tipRepairPendingBaseBackoff, 20*time.Millisecond, "the idle worker repairs the recovered subtree once the back-off has passed") } + +// erroringSubtreeStore fails the next fail reads of one subtree with err. +type erroringSubtreeStore struct { + blob.Store + key []byte + err error + fail atomic.Int32 +} + +func (s *erroringSubtreeStore) GetIoReader(ctx context.Context, key []byte, fileType fileformat.FileType, opts ...blobopts.FileOption) (io.ReadCloser, error) { + if bytes.Equal(key, s.key) && s.fail.Add(-1) >= 0 { + return nil, s.err + } + + return s.Store.GetIoReader(ctx, key, fileType, opts...) +} + +// erroringReads wraps store so the next n reads of the block's subtree idx fail with err. +func (f *overtakeFixture) erroringReads(store blob.Store, idx int, n int32, err error) *erroringSubtreeStore { + s := &erroringSubtreeStore{Store: store, key: f.blockB.Subtrees[idx][:], err: err} + s.fail.Store(n) + + return s +} + +// TestRepairOvertakenTips_StorageErrorOverNotFoundRetried: a storage error is the store +// failing while it looked, whatever it wraps, so one over a not-found must not get the +// subtree skipped as gone (the subtreeAbsent rule). The step is retried once the store is +// back, and the subtree is repaired. +func TestRepairOvertakenTips_StorageErrorOverNotFoundRetried(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + storageErr := errors.NewStorageError("store failed while reading", errors.NewNotFoundError("not found")) + bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 2*tipRepairSubtreeReadAttempts, storageErr) + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].done }) + + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once the store is back, not skipped") +} + +// TestRepairOvertakenTips_PersistentStoreErrorSetAside: a subtree whose reads keep failing +// with any error, not only a deserialize failure, is set aside after tipRepairMaxAttempts +// failed loads, so the later subtrees are repaired and the job does not start over from the +// first subtree each time it is deferred. +func TestRepairOvertakenTips_PersistentStoreErrorSetAside(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + _ = f.fakeTipRepairClock(bv) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewStorageError("injected subtree store failure")) + + settleTipRepair(t, bv, func() bool { return f.unminedSince(t, f.later) != 0 }) + + require.NotZero(t, f.unminedSince(t, f.later), "the later subtree is repaired") + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, b.done) + require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending, "the failing subtree is read again later") +} From e9d2b81fd458eb05306e53ed69d4432e36a88fcf Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 11:00:30 +0100 Subject: [PATCH 19/33] fix(blockvalidation): clear deferred when a block is back on the chain, warn only when txs were re-marked A block with subtrees set aside that came back onto the chain kept deferred, so if it left again its new repair waited behind every other block. An unmark job that re-marked nothing, such as one for a fork block the restart seed tracks, no longer logs a WARN that the block's txs had been marked mined. MockStore.GetLastNBlocks returns no blocks for n <= 0 instead of panicking. --- .../blockvalidation/overtaken_tip_repair.go | 17 ++++- .../settxmined_overtaken_test.go | 65 +++++++++++++++++++ stores/blockchain/mock.go | 4 ++ stores/blockchain/mock_last_n_blocks_test.go | 20 ++++++ 4 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 stores/blockchain/mock_last_n_blocks_test.go diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index a22545bac5..854e090b7f 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -468,9 +468,14 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b delete(tracked, hash) case onMainChain[b.id] && b.pending != nil: - // Back on the chain: if it leaves again, all of it needs repairing. - u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.clearPending() }) + // Back on the chain: if it leaves again, all of it needs repairing, and not behind + // the other blocks for subtrees it could not read last time. + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { + e.clearPending() + e.deferred = false + }) b.clearPending() + b.deferred = false tracked[hash] = b case onMainChain[b.id] && b.done && !b.stamped: @@ -750,7 +755,13 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { } }) - u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain after its txs were marked mined, marked %d of %d txs as not on longest chain", job.hash.String(), job.marked, job.total) + // A block none of whose txs needed re-marking, such as a fork block the restart seed + // tracks that was never on the chain, is not worth a warning. + if job.marked > 0 { + u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain after its txs were marked mined, marked %d of %d txs as not on longest chain", job.hash.String(), job.marked, job.total) + } else { + u.logger.Infof("[repairOvertakenTips][%s] block is off the longest chain, none of its %d txs needed marking as not on longest chain", job.hash.String(), job.total) + } case remineJob: u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index dc24fd7289..199693adb0 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -6,6 +6,7 @@ import ( "io" "net/url" "slices" + "strings" "sync" "sync/atomic" "testing" @@ -1667,3 +1668,67 @@ func TestRepairOvertakenTips_PersistentStoreErrorSetAside(t *testing.T) { require.False(t, b.done) require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending, "the failing subtree is read again later") } + +// TestRepairOvertakenTips_BackOnChainClearsDeferred: a block whose subtrees were set aside +// and which is back on the chain must not stay deferred. If it leaves again, its new repair +// has nothing to do with the old unreadable subtrees and must not wait behind every other +// block. +func TestRepairOvertakenTips_BackOnChainClearsDeferred(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, 1<<30) + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].deferred, "precondition: the block is deferred") + + f.extendB(t) + + _, _ = bv.tipRepairSlice(f.ctx, bv.blockchainClient) + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].deferred) +} + +// warnRecorder records the format of every overtaken-tip repair Warnf; the background +// workers winding down log their own. +type warnRecorder struct { + ulogger.TestLogger + mu sync.Mutex + warns []string +} + +func (l *warnRecorder) Warnf(format string, _ ...interface{}) { + if !strings.HasPrefix(format, "[repairOvertakenTips]") { + return + } + + l.mu.Lock() + defer l.mu.Unlock() + + l.warns = append(l.warns, format) +} + +// TestFinishJob_WarnsOnlyWhenTxsMarked: an unmark job that re-marked none of the block's +// txs, such as one for a fork block the restart seed tracks that was never on the chain, is +// not reported as a block that left the chain after its txs were marked mined. +func TestFinishJob_WarnsOnlyWhenTxsMarked(t *testing.T) { + f := newOvertakeFixture(t) + logger := &warnRecorder{} + + ctx, stop := context.WithCancel(f.ctx) + bv := NewBlockValidation(ctx, logger, f.tSettings, f.blockchainClient, f.subtreeStore, f.txStore, f.store, nil, f.subtreeValidationClient) + stop() + bv.backgroundTasks.Wait() + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + hash := *f.blockB.Hash() + b := bv.minedOnChain.snapshot()[hash] + + bv.finishJob(&tipRepairJob{kind: unmarkJob, hash: hash, id: b.id, gen: b.gen, total: 3}) + require.Empty(t, logger.warns, "no warning when no tx was marked") + + bv.finishJob(&tipRepairJob{kind: unmarkJob, hash: hash, id: b.id, gen: b.gen, marked: 2, total: 3}) + require.Len(t, logger.warns, 1, "a warning when txs were marked") +} diff --git a/stores/blockchain/mock.go b/stores/blockchain/mock.go index df80d7a9dd..6c505b5180 100644 --- a/stores/blockchain/mock.go +++ b/stores/blockchain/mock.go @@ -332,6 +332,10 @@ func (m *MockStore) AssignBlockID(_ context.Context, blockHash *chainhash.Hash) // GetLastNBlocks returns up to n blocks, highest first. Without includeOrphans only the best // block's chain is returned; fromHeight, when non-zero, caps the height. func (m *MockStore) GetLastNBlocks(ctx context.Context, n int64, includeOrphans bool, fromHeight uint32) ([]*model.BlockInfo, error) { + if n <= 0 { + return []*model.BlockInfo{}, nil + } + m.mu.RLock() defer m.mu.RUnlock() diff --git a/stores/blockchain/mock_last_n_blocks_test.go b/stores/blockchain/mock_last_n_blocks_test.go new file mode 100644 index 0000000000..171b7a76f0 --- /dev/null +++ b/stores/blockchain/mock_last_n_blocks_test.go @@ -0,0 +1,20 @@ +package blockchain + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestMockStore_GetLastNBlocksNonPositiveN: a count of zero or less returns no blocks, as +// LIMIT does in the SQL store, instead of panicking on a negative slice capacity. +func TestMockStore_GetLastNBlocksNonPositiveN(t *testing.T) { + m := NewMockStore() + + for _, n := range []int64{0, -1} { + infos, err := m.GetLastNBlocks(context.Background(), n, true, 0) + require.NoError(t, err) + require.Empty(t, infos) + } +} From 42fbb9a177bad6013f3cc6b6c6490f5acd3083ba Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 11:09:19 +0100 Subject: [PATCH 20/33] fix(blockvalidation): repair finished overtaken blocks again after a reorg, keep stamped across a new write An unmark job leaves a tx mined when another block holding it is on the chain. When that block reached the chain only through its child, no setTxMined tracked it, so once a third chain overtook it the tx stayed mined off chain. When the last best block leaves the chain, every finished off-chain block is now read again. track() reset stamped for a new on-chain write. If that write failed before clearing the re-marked txs, nothing would mark them mined again; stamped now stays until the remine job runs, which writes nothing when the write did complete. --- services/blockvalidation/BlockValidation.go | 1 + .../blockvalidation/overtaken_tip_repair.go | 22 +++++++- .../settxmined_overtaken_test.go | 52 ++++++++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 67f929041b..7089f9697e 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -332,6 +332,7 @@ type BlockValidation struct { // tipRepairMu), so a best block that only extends the chain costs one call. tipRepairMainChain map[uint32]bool tipRepairBestHash chainhash.Hash + tipRepairBestID uint32 // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. tipRepairRetry atomic.Bool // tipRepairWake requests a slice when the earliest pass over subtrees set aside comes diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 854e090b7f..cba4295eaf 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -177,8 +177,10 @@ func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32) { t.mu.Lock() defer t.mu.Unlock() + // stamped stays: if this write fails before clearing the re-marked txs again, the remine + // job is what marks them mined, and a remine over txs already mined writes nothing. t.nextGen++ - t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, failures: t.blocks[hash].failures} + t.blocks[hash] = minedOnChainBlock{id: id, height: height, gen: t.nextGen, failures: t.blocks[hash].failures, stamped: t.blocks[hash].stamped} } // seed watches a block found by the restart seed. A block already watched keeps its state. @@ -450,11 +452,19 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b } } + hadMainChain, prevBestID := u.tipRepairMainChain != nil, u.tipRepairBestID + onMainChain, bestHeight, err := u.mainChainIDsLocked(ctx, blockchainClient) if err != nil { return false, err } + // The last best block left the chain. A finished unmark job left a tx mined when another + // block holding it was on the chain, and that block may have just left with it, untracked + // if it reached the chain only through a child. Every finished off-chain block is read + // again. + reorged := hadMainChain && !onMainChain[prevBestID] + tracked := u.minedOnChain.snapshot() for hash, b := range tracked { @@ -478,6 +488,11 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b b.deferred = false tracked[hash] = b + case reorged && !onMainChain[b.id] && b.done: + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.done = false }) + b.done = false + tracked[hash] = b + case onMainChain[b.id] && b.done && !b.stamped: // Back on the chain and nothing was re-marked: nothing to undo, but if it // leaves again it needs repairing again. @@ -589,6 +604,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie // blocks before the set is consulted for them. u.tipRepairMainChain[ids[0]] = true u.tipRepairBestHash = bestHash + u.tipRepairBestID = ids[0] return u.tipRepairMainChain, bestMeta.Height, nil } @@ -606,6 +622,10 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie u.tipRepairMainChain = onMainChain u.tipRepairBestHash = bestHash + if len(ids) > 0 { + u.tipRepairBestID = ids[0] + } + return onMainChain, bestMeta.Height, nil } diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 199693adb0..d1821bbdee 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -857,7 +857,7 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { b := tracker.snapshot()[hash] require.NotEqual(t, gen, b.gen, "a new write is a new generation; a repair in progress for the old one is abandoned") require.False(t, b.done, "the new write clears the txs again, so the block needs repairing again") - require.False(t, b.stamped) + require.True(t, b.stamped, "kept in case the new write fails before clearing the re-marked txs") tracker.update(hash, gen, func(e *minedOnChainBlock) { e.done = true }) require.False(t, tracker.snapshot()[hash].done, "an update for the old generation is ignored") @@ -1732,3 +1732,53 @@ func TestFinishJob_WarnsOnlyWhenTxsMarked(t *testing.T) { bv.finishJob(&tipRepairJob{kind: unmarkJob, hash: hash, id: b.id, gen: b.gen, marked: 2, total: 3}) require.Len(t, logger.warns, 1, "a warning when txs were marked") } + +// TestRepairOvertakenTips_ThirdChainReopensDoneBlock: B's repair left shared mined because +// B', which also holds it, was on the chain. B' got there through its child C', so no +// setTxMined of B' tracks it. When a third chain overtakes B' -> C', shared is in no block +// on the chain any more, and B must be repaired again to mark it. +func TestRepairOvertakenTips_ThirdChainReopensDoneBlock(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockC2.Hash())) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "precondition: B is repaired") + require.Zero(t, f.unminedSince(t, f.shared), "precondition: shared is mined in B' on the chain") + + time.Sleep(10 * time.Millisecond) + + parent := f.tSettings.ChainCfgParams.GenesisHash + + for height := uint32(1); height <= 3; height++ { + block := coinbaseOnlyBlock(t, parent, f.nBits, height, 0xf0+byte(height)) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, block, "test", blockchainoptions.WithSubtreesSet(true))) + parent = block.Hash() + } + + require.False(t, f.onChain(t, f.blockBID), "precondition: the third chain overtook both") + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.shared), "shared is in no block on the chain and must look unmined") +} + +// TestMinedOnChainTracker_TrackKeepsStamped: a new on-chain write of a block whose txs were +// re-marked not on the longest chain must not forget that they were: if the write fails +// before clearing them, the remine job is the only thing left to mark them mined again. +func TestMinedOnChainTracker_TrackKeepsStamped(t *testing.T) { + tracker := newMinedOnChainTracker() + hash := chainhash.HashH([]byte("b")) + + tracker.track(hash, 1, 1) + + gen := tracker.snapshot()[hash].gen + tracker.update(hash, gen, func(e *minedOnChainBlock) { e.stamped = true }) + + tracker.track(hash, 1, 1) + require.True(t, tracker.snapshot()[hash].stamped) +} From 1c212f27779239551da78f277b71e47e06cfc370 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 11:13:12 +0100 Subject: [PATCH 21/33] refactor(utxo/sql): one lock-error retry helper for reads and writes; correct the ErrBlockParentNotMined comment retryWriteOnLockError was a copy of retryReadOnLockError; both are now retryOnLockError, which takes the attempt count and a label for its log. The in-flight setTxMined that ErrBlockParentNotMined reports belongs to another BlockValidation in the process, not this one; the comment said its write would have this block repaired again. --- services/blockvalidation/BlockValidation.go | 5 ++- stores/utxo/sql/parent_outputs.go | 24 +---------- stores/utxo/sql/parent_outputs_test.go | 8 ++-- stores/utxo/sql/setmined_lock_retry_test.go | 8 ++-- stores/utxo/sql/sql.go | 44 ++++++++++----------- 5 files changed, 35 insertions(+), 54 deletions(-) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 7089f9697e..d125fb89f8 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1714,8 +1714,9 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // provenance is attached; the p2p consumer falls back to its peer map. return u.markBlockAsInvalid(ctx, block, "contains transactions already on our chain: "+err.Error(), "", "") } else if errors.Is(err, errors.ErrBlockParentNotMined) { - // Another call for this block is in flight. Its tracking stays, and that write - // has the block repaired again when it completes. + // Another call for this block is in flight. This worker is the only caller here, + // so it is another BlockValidation in this process, with its own tracker. This + // tracker keeps the block as track() left it, not done, so its repair runs again. u.logger.Warnf("[setTxMined][%s] skipping, already in progress of setMined for block", block.Hash().String()) return nil } diff --git a/stores/utxo/sql/parent_outputs.go b/stores/utxo/sql/parent_outputs.go index aa97793a9a..7ae24b58f7 100644 --- a/stores/utxo/sql/parent_outputs.go +++ b/stores/utxo/sql/parent_outputs.go @@ -2,13 +2,12 @@ package sql import ( "context" - "time" + "fmt" "github.com/bsv-blockchain/go-bt/v2/bscript" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/stores/utxo" - "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util" "github.com/bsv-blockchain/teranode/util/tracing" ) @@ -111,7 +110,7 @@ const parentOutputsReadRetries = 3 func (s *Store) parentOutputsChunkWithRetry(ctx context.Context, chunk []outpointPair) (map[utxo.Outpoint]utxo.ParentOutput, error) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryReadOnLockError(ctx, s.logger, len(chunk), func() error { + err := retryOnLockError(ctx, s.logger, parentOutputsReadRetries, fmt.Sprintf("[ParentOutputsForValidation] chunk of %d outpoints", len(chunk)), func() error { var err error found, err = s.parentOutputsChunk(ctx, chunk) @@ -122,25 +121,6 @@ func (s *Store) parentOutputsChunkWithRetry(ctx context.Context, chunk []outpoin return found, err } -// retryReadOnLockError runs read up to parentOutputsReadRetries times while it -// fails with an error isDeadlock recognises, and returns its last error. -func retryReadOnLockError(ctx context.Context, logger ulogger.Logger, outpoints int, read func() error) error { - for attempt := 1; ; attempt++ { - err := read() - if err == nil || !isDeadlock(err) || attempt >= parentOutputsReadRetries { - return err - } - - logger.Warnf("[ParentOutputsForValidation] lock error (attempt %d/%d), retrying chunk of %d outpoints: %v", attempt, parentOutputsReadRetries, outpoints, err) - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Duration(attempt) * 10 * time.Millisecond): - } - } -} - // parentOutputsChunk runs one statement for chunk and returns an answer for // every outpoint whose transaction exists. Outpoints with no transaction row are // absent from the map. diff --git a/stores/utxo/sql/parent_outputs_test.go b/stores/utxo/sql/parent_outputs_test.go index 23d12fb32e..7f8a89ba38 100644 --- a/stores/utxo/sql/parent_outputs_test.go +++ b/stores/utxo/sql/parent_outputs_test.go @@ -82,7 +82,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryReadOnLockError(ctx, ulogger.TestLogger{}, len(chunk), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ if attempts < parentOutputsReadRetries { return lockErr @@ -106,7 +106,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryReadOnLockError(ctx, ulogger.TestLogger{}, len(chunk), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ return lockErr }) @@ -118,7 +118,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewStorageError("no such table: outputs") - err := retryReadOnLockError(ctx, ulogger.TestLogger{}, len(chunk), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ return other }) @@ -130,7 +130,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryReadOnLockError(cctx, ulogger.TestLogger{}, len(chunk), func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ cancel() diff --git a/stores/utxo/sql/setmined_lock_retry_test.go b/stores/utxo/sql/setmined_lock_retry_test.go index 8f1cc05051..70f698bdb4 100644 --- a/stores/utxo/sql/setmined_lock_retry_test.go +++ b/stores/utxo/sql/setmined_lock_retry_test.go @@ -38,7 +38,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { var blockIDs map[chainhash.Hash][]uint32 - err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ if attempts < setMinedMultiRetries { return lockErr @@ -58,7 +58,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ return lockErr }) @@ -70,7 +70,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewTxNotFoundError("transaction not found") - err := retryWriteOnLockError(ctx, ulogger.TestLogger{}, len(hashes), func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ return other }) @@ -82,7 +82,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryWriteOnLockError(cctx, ulogger.TestLogger{}, len(hashes), func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ cancel() diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 62e0cd84ff..007de6632f 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -2421,6 +2421,27 @@ func isDeadlock(err error) bool { strings.Contains(msg, "(SQLSTATE "+usql.PgErrDeadlockDetected+")") } +// retryOnLockError runs op up to attempts times while it fails with an error +// isDeadlock recognises, and returns its last error. what names the operation +// in the warning logged before each retry. op must be safe to run again after +// such a failure: a failed statement or rolled-back transaction holds no locks. +func retryOnLockError(ctx context.Context, logger ulogger.Logger, attempts int, what string, op func() error) error { + for attempt := 1; ; attempt++ { + err := op() + if err == nil || !isDeadlock(err) || attempt >= attempts { + return err + } + + logger.Warnf("%s: lock error (attempt %d/%d), retrying: %v", what, attempt, attempts, err) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Duration(attempt) * 10 * time.Millisecond): + } + } +} + // sendSpendBatch is the batcher callback that processes a batch of spend operations // in a single DB transaction. Called by the go-batcher when the batch is full or // the timeout fires. Same SQL for both PostgreSQL and SQLite. @@ -3636,7 +3657,7 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min var chunkResult map[chainhash.Hash][]uint32 - err := retryWriteOnLockError(ctx, s.logger, end-i, func() error { + err := retryOnLockError(ctx, s.logger, setMinedMultiRetries, fmt.Sprintf("[SetMinedMulti] chunk of %d txs", end-i), func() error { var err error chunkResult, err = s.setMinedMultiChunk(ctx, hashes[i:end], minedBlockInfo) @@ -3659,27 +3680,6 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min // when the database refuses it with a lock error. const setMinedMultiRetries = 3 -// retryWriteOnLockError runs write up to setMinedMultiRetries times while it -// fails with an error isDeadlock recognises, and returns its last error. A -// chunk is one transaction, rolled back when it fails, so running it again is -// safe. -func retryWriteOnLockError(ctx context.Context, logger ulogger.Logger, txs int, write func() error) error { - for attempt := 1; ; attempt++ { - err := write() - if err == nil || !isDeadlock(err) || attempt >= setMinedMultiRetries { - return err - } - - logger.Warnf("[SetMinedMulti] lock error (attempt %d/%d), retrying chunk of %d txs: %v", attempt, setMinedMultiRetries, txs, err) - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(time.Duration(attempt) * 10 * time.Millisecond): - } - } -} - // setMinedMultiChunk processes a single chunk of hashes (up to maxINClauseSize). // Uses portable IN ($1,$2,...,$N) clauses that work on both PostgreSQL and SQLite. func (s *Store) setMinedMultiChunk(ctx context.Context, hashes []*chainhash.Hash, minedBlockInfo utxo.MinedBlockInfo) (map[chainhash.Hash][]uint32, error) { From aa96a33d098a5bc7c5857cd0c15521b9b5542e43 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 11:47:59 +0100 Subject: [PATCH 22/33] fix(blockvalidation): track blocks a reorg brings onto the chain, back off from a pass's end, never set a remine's subtree aside Instead of re-reading every finished off-chain block after a reorg, the repair tracks the window's main-chain blocks when the last best block has left the chain. A block that reached the chain only through a child is then tracked, and when it leaves, its own repair marks the txs other repairs left mined because it was on the chain, whether those repairs finished, are still running, or wait on set-aside subtrees. - The back-off's height is the best height when a pass ends, not when it started, so a pass that spans a block does not start the next at once. - A remine job never sets a subtree aside: finishing would drop the re-mine of its txs. Its load errors fail the step and it is retried. - A completed on-chain write clears stamped; only a failed one leaves the remine owed. - No ids for the best block is a failed read, not an empty chain. - A deferred block on the chain with no remine owed is no longer deferred. - finishJob warns when subtrees were set aside even if nothing was marked. - The setMined worker stops the wake timer when it exits. - retryOnLockError formats its label only when it retries. --- services/blockvalidation/BlockValidation.go | 7 +- .../blockvalidation/overtaken_tip_repair.go | 75 +++--- .../settxmined_overtaken_test.go | 218 ++++++++++++++++-- stores/blockchain/mock_last_n_blocks_test.go | 4 +- stores/utxo/sql/parent_outputs.go | 3 +- stores/utxo/sql/parent_outputs_test.go | 8 +- stores/utxo/sql/setmined_lock_retry_test.go | 8 +- stores/utxo/sql/sql.go | 15 +- 8 files changed, 275 insertions(+), 63 deletions(-) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index d125fb89f8..73cc3b85db 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -776,6 +776,11 @@ func (u *BlockValidation) start(ctx context.Context) error { go func() { defer u.backgroundTasks.Done() defer u.logger.Infof("[BlockValidation:start] setMinedChan worker stopped") + defer func() { + if u.tipRepairWake != nil { + u.tipRepairWake.Stop() + } + }() for { // One slice of the overtaken-tip repair between blocks, when one is pending. @@ -1727,7 +1732,7 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // The write is complete, whichever onLongestChain this attempt used. Ask for a slice: a // chain change that happened while the write was in flight would otherwise wait for the // next one. - if u.minedOnChain.markWritten(*blockHash) { + if u.minedOnChain.markWritten(*blockHash, onLongestChain) { u.signalTipRepair() } diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index cba4295eaf..32d15b39e4 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -202,8 +202,9 @@ func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32) { // markWritten records that a setTxMined write of the block has completed and reports // whether the block is tracked, in which case the caller asks for a repair slice. The write -// may have cleared txs after a repair read them, so a finished repair runs again. -func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { +// may have cleared txs after a repair read them, so a finished repair runs again. A write as +// on the longest chain cleared every tx of the block, so no remine is owed after it. +func (t *minedOnChainTracker) markWritten(hash chainhash.Hash, onLongestChain bool) bool { if t == nil { return false } @@ -218,6 +219,11 @@ func (t *minedOnChainTracker) markWritten(hash chainhash.Hash) bool { b.done = false b.clearPending() + + if onLongestChain { + b.stamped = false + } + t.blocks[hash] = b return true @@ -307,8 +313,8 @@ type tipRepairJob struct { subtreeFailures int // unreadable holds the subtrees set aside because they could not be read. unreadable []*chainhash.Hash - // startHeight is the best height when the job started. - startHeight uint32 + // height is the best height at the job's latest step. + height uint32 } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -459,11 +465,19 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b return false, err } - // The last best block left the chain. A finished unmark job left a tx mined when another - // block holding it was on the chain, and that block may have just left with it, untracked - // if it reached the chain only through a child. Every finished off-chain block is read - // again. - reorged := hadMainChain && !onMainChain[prevBestID] + // The last best block left the chain. An unmark job leaves a tx mined while another block + // holding it is on the chain, so every block on the chain must be tracked, to have its + // own repair when it leaves. Blocks written on the chain by setTxMined are; one that + // reached the chain only through a child, its own setTxMined having run while it was off + // the chain, is tracked here. + if hadMainChain && !onMainChain[prevBestID] { + headers, metas, err := blockchainClient.GetBlockHeaders(ctx, &u.tipRepairBestHash, tipRepairWindow) + if err != nil { + return false, errors.NewServiceError("[repairOvertakenTips] failed to get main chain headers after a reorg", err) + } + + u.trackMinedHeaders(headers, metas) + } tracked := u.minedOnChain.snapshot() @@ -488,9 +502,11 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b b.deferred = false tracked[hash] = b - case reorged && !onMainChain[b.id] && b.done: - u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.done = false }) - b.done = false + case onMainChain[b.id] && b.deferred && !b.stamped: + // On the chain with no remine owed: the job that failed is moot, and if the block + // leaves again its repair must not wait behind every other block. + u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.deferred = false }) + b.deferred = false tracked[hash] = b case onMainChain[b.id] && b.done && !b.stamped: @@ -514,6 +530,8 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b return true, nil } + job.height = bestHeight + if err = u.jobStep(ctx, blockchainClient, job, onMainChain); err == nil { job.failures = 0 return true, nil @@ -609,8 +627,10 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie return u.tipRepairMainChain, bestMeta.Height, nil } + // No ids is a failed read, not a chain with no blocks: taken as one, every tracked block + // would look off the chain. ids, err := blockchainClient.GetBlockHeaderIDs(ctx, &bestHash, tipRepairWindow) - if err != nil { + if err != nil || len(ids) == 0 { return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get main chain block ids", err) } @@ -621,10 +641,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie u.tipRepairMainChain = onMainChain u.tipRepairBestHash = bestHash - - if len(ids) > 0 { - u.tipRepairBestID = ids[0] - } + u.tipRepairBestID = ids[0] return onMainChain, bestMeta.Height, nil } @@ -650,21 +667,21 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch subtrees = b.pending } - u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees, startHeight: bestHeight} + u.tipRepairJob = &tipRepairJob{kind: kind, hash: hash, id: b.id, gen: b.gen, subtrees: subtrees, height: bestHeight} return nil } // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone -// is skipped with an error log. One that still cannot be loaded after tipRepairMaxAttempts -// failed loads, whatever the error, is set aside, so the rest of the block is repaired: a -// run of short reads looks the same as a corrupt file, and a store that stays down must not -// hold up the later subtrees. An unmark job's set-aside subtrees -// are read again by the block's next unmark job (see finishJob). A remine job does not -// come back to them, which is the safe direction: their txs stay flagged unmined until -// fixUnminedSinceInconsistencies clears them. Until then a load error fails the step, to be -// retried. +// is skipped with an error log. For an unmark job, one that still cannot be loaded after +// tipRepairMaxAttempts failed loads, whatever the error, is set aside, so the rest of the +// block is repaired: a run of short reads looks the same as a corrupt file, and a store that +// stays down must not hold up the later subtrees. The block's next unmark job reads the +// set-aside subtrees again (see finishJob). A remine job never sets one aside: finishing +// would drop the re-mine of its txs, so its load errors fail the step and the job is +// retried, deferred behind other blocks once it keeps failing. Until a subtree is set aside, +// a load error fails the step, to be retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { if job.nextSubtree >= len(job.subtrees) { @@ -681,7 +698,7 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha switch { case isPermanentRepairError(err): u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) - case ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: + case job.kind == unmarkJob && ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) job.unreadable = append(job.unreadable, subtreeHash) default: @@ -771,13 +788,13 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { if !e.done { e.passes++ e.nextPass = u.minedOnChain.now().Add(min(tipRepairPendingBaseBackoff< 0 { + if job.marked > 0 || len(job.unreadable) > 0 { u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain after its txs were marked mined, marked %d of %d txs as not on longest chain", job.hash.String(), job.marked, job.total) } else { u.logger.Infof("[repairOvertakenTips][%s] block is off the longest chain, none of its %d txs needed marking as not on longest chain", job.hash.String(), job.total) diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index d1821bbdee..71d19f75ed 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -843,7 +843,7 @@ func TestMinedOnChainTracker_NewWriteStartsOver(t *testing.T) { hash := chainhash.HashH([]byte("b")) tracker.track(hash, 1, 1) - require.True(t, tracker.markWritten(hash)) + require.True(t, tracker.markWritten(hash, true)) gen := tracker.snapshot()[hash].gen tracker.update(hash, gen, func(e *minedOnChainBlock) { @@ -869,7 +869,7 @@ func TestMinedOnChainTracker_Nil(t *testing.T) { tracker.track(hash, 1, 1) tracker.seed(hash, 1, 1) - require.False(t, tracker.markWritten(hash)) + require.False(t, tracker.markWritten(hash, true)) tracker.update(hash, 1, func(e *minedOnChainBlock) { e.done = true }) tracker.drop(hash) require.True(t, tracker.empty()) @@ -1135,7 +1135,7 @@ func TestRepairOvertakenTips_StuckJobYields(t *testing.T) { require.False(t, f.onChain(t, forkMeta.ID)) bv.minedOnChain.track(*fork.Hash(), forkMeta.ID, forkMeta.Height) - require.True(t, bv.minedOnChain.markWritten(*fork.Hash())) + require.True(t, bv.minedOnChain.markWritten(*fork.Hash(), true)) for i := 0; i < 3*tipRepairMaxAttempts; i++ { _, _ = bv.tipRepairSlice(f.ctx, bv.blockchainClient) @@ -1448,7 +1448,7 @@ func TestMinedOnChainTracker_WriteClearsPending(t *testing.T) { gen := tracker.snapshot()[hash].gen tracker.update(hash, gen, func(e *minedOnChainBlock) { e.pending = []*chainhash.Hash{&subtree} }) - require.True(t, tracker.markWritten(hash)) + require.True(t, tracker.markWritten(hash, true)) require.Nil(t, tracker.snapshot()[hash].pending) } @@ -1580,7 +1580,15 @@ func TestRepairOvertakenTips_PendingPassWakesIdleWorker(t *testing.T) { reads := f.shortReads(f.subtreeStore, 0, 0) f.subtreeStore = reads - _ = f.newRunningBlockValidation(f.ctx) + + // Registered after the fixture's cleanups, so it runs first: the worker stops before the + // stores it uses are closed. + ctx, stop := context.WithCancel(f.ctx) + bv := f.newRunningBlockValidation(ctx) + t.Cleanup(func() { + stop() + bv.backgroundTasks.Wait() + }) require.Eventually(t, func() bool { subscribers, err := f.blockchainClient.GetSubscribers(f.ctx) @@ -1731,22 +1739,67 @@ func TestFinishJob_WarnsOnlyWhenTxsMarked(t *testing.T) { bv.finishJob(&tipRepairJob{kind: unmarkJob, hash: hash, id: b.id, gen: b.gen, marked: 2, total: 3}) require.Len(t, logger.warns, 1, "a warning when txs were marked") + + bv.finishJob(&tipRepairJob{kind: unmarkJob, hash: hash, id: b.id, gen: b.gen, unreadable: f.blockB.Subtrees[:1]}) + require.Len(t, logger.warns, 2, "a warning when subtrees were set aside, though none was marked yet") } -// TestRepairOvertakenTips_ThirdChainReopensDoneBlock: B's repair left shared mined because -// B', which also holds it, was on the chain. B' got there through its child C', so no -// setTxMined of B' tracks it. When a third chain overtakes B' -> C', shared is in no block -// on the chain any more, and B must be repaired again to mark it. -func TestRepairOvertakenTips_ThirdChainReopensDoneBlock(t *testing.T) { +// blockWithShared returns a block on parent whose one subtree holds shared, with its +// subtree stored, as the competing chain's block that also mines shared. +func (f *overtakeFixture) blockWithShared(t *testing.T, parent *chainhash.Hash, height uint32, tag byte) *model.Block { + t.Helper() + + coinbaseTx := bt.NewTx() + require.NoError(t, coinbaseTx.From("0000000000000000000000000000000000000000000000000000000000000000", 0xffffffff, "", 0)) + coinbaseTx.Inputs[0].UnlockingScript = bscript.NewFromBytes([]byte{0x03, byte(height), 0x00, 0x00, tag, '/', 'T', 'e', 's', 't'}) + require.NoError(t, coinbaseTx.AddP2PKHOutputFromAddress("1MUMxUTXcPQ1kAqB7MtJWneeAwVW4cHzzp", 50*100000000)) + + subtree, err := subtreepkg.NewTreeByLeafCount(2) + require.NoError(t, err) + require.NoError(t, subtree.AddCoinbaseNode()) + require.NoError(t, subtree.AddNode(*f.shared, 100, 0)) + + subtreeBytes, err := subtree.Serialize() + require.NoError(t, err) + require.NoError(t, f.subtreeStore.Set(f.ctx, subtree.RootHash()[:], fileformat.FileTypeSubtree, subtreeBytes)) + + replicated := subtree.Duplicate() + replicated.ReplaceRootNode(coinbaseTx.TxIDChainHash(), 0, uint64(coinbaseTx.Size())) //nolint:gosec + + header := mineBlockHeader(t, parent, replicated.RootHash(), f.nBits) + + block, err := model.NewBlock(header, coinbaseTx, []*chainhash.Hash{subtree.RootHash()}, uint64(subtree.Length()), uint64(coinbaseTx.Size())+100, height, 0) //nolint:gosec + require.NoError(t, err) + + return block +} + +// TestRepairOvertakenTips_ThirdChainRepairsBlockJoinedThroughAChild: B's repair leaves +// shared mined because B', which also holds it, is on the chain. B' got there through its +// child C', so no on-chain setTxMined of B' tracked it. When a third chain overtakes +// B' -> C', shared is in no block on the chain any more, and the repair of B' itself must mark it. +func TestRepairOvertakenTips_ThirdChainRepairsBlockJoinedThroughAChild(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) - blockC2 := f.overtakeB(t) - require.NoError(t, bv.setTxMinedStatus(f.ctx, blockC2.Hash())) - _, err := drainTipRepair(t, bv) require.NoError(t, err) + + time.Sleep(10 * time.Millisecond) + + blockB2 := f.blockWithShared(t, f.tSettings.ChainCfgParams.GenesisHash, 1, 0xb2) + blockC2 := coinbaseOnlyBlock(t, blockB2.Hash(), f.nBits, 2, 0xc2) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockB2, "test", blockchainoptions.WithSubtreesSet(true))) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockC2, "test", blockchainoptions.WithSubtreesSet(true))) + require.False(t, f.onChain(t, f.blockBID), "precondition: the competing chain overtook B") + + _, metaB2, err := f.blockchainClient.GetBlockHeader(f.ctx, blockB2.Hash()) + require.NoError(t, err) + f.mineSharedIn(t, metaB2.ID) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "precondition: B is repaired") require.Zero(t, f.unminedSince(t, f.shared), "precondition: shared is mined in B' on the chain") @@ -1760,7 +1813,7 @@ func TestRepairOvertakenTips_ThirdChainReopensDoneBlock(t *testing.T) { parent = block.Hash() } - require.False(t, f.onChain(t, f.blockBID), "precondition: the third chain overtook both") + require.False(t, f.onChain(t, metaB2.ID), "precondition: the third chain overtook B'") _, err = drainTipRepair(t, bv) require.NoError(t, err) @@ -1782,3 +1835,140 @@ func TestMinedOnChainTracker_TrackKeepsStamped(t *testing.T) { tracker.track(hash, 1, 1) require.True(t, tracker.snapshot()[hash].stamped) } + +// TestRepairOvertakenTips_ReorgTracksBlocksJoiningThroughAChild: B' reaches the chain only +// through its child C', so no on-chain setTxMined of B' tracks it. The reorg that brings it +// in must, so that when B' leaves, its own repair marks the txs other blocks' repairs left +// mined because B' was on the chain, whether those repairs finished, are still running, or +// wait on subtrees set aside. +func TestRepairOvertakenTips_ReorgTracksBlocksJoiningThroughAChild(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + + blockC2 := f.overtakeB(t) + + _, err = bv.tipRepairSlice(f.ctx, bv.blockchainClient) + require.NoError(t, err) + + _, tracked := bv.minedOnChain.snapshot()[*blockC2.Header.HashPrevBlock] + require.True(t, tracked, "B' is tracked once the reorg brings it onto the chain") +} + +// TestRepairOvertakenTips_BackOffFromPassEnd: a pass over a subtree that keeps failing spans +// several blocks. The next pass is due a block after the pass ended, not after it started, +// or it starts again at once and a hopeless subtree holds the repair's one job slot. +func TestRepairOvertakenTips_BackOffFromPassEnd(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + _ = f.fakeTipRepairClock(bv) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + store := f.shortReads(f.subtreeStore, 0, 1<<30) + bv.subtreeStore = store + + _, err := drainTipRepair(t, bv) + require.Error(t, err, "precondition: the pass is under way, failing") + + f.extendWinning(t, blockC2) + + settleTipRepair(t, bv, func() bool { + return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil && bv.tipRepairJob == nil + }) + require.NotNil(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending, "precondition: the subtree is set aside") + + require.Equal(t, 1, bv.minedOnChain.snapshot()[*f.blockB.Hash()].passes, + "the block that arrived during the pass does not start another one as soon as it ends") + + left := store.short.Load() + + for i := 0; i < 5; i++ { + _, _ = drainTipRepair(t, bv) + } + + require.Equal(t, left, store.short.Load(), "no new pass before a block arrives after the last one ended") +} + +// TestRepairOvertakenTips_RemineNeverSetsAside: a remine job whose subtree keeps failing to +// load must not set it aside and finish: that drops the re-mine of its txs for good. The +// block stays stamped and the remine is retried. +func TestRepairOvertakenTips_RemineNeverSetsAside(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped, "precondition: B's txs were re-marked") + + f.extendB(t) + + bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewStorageError("injected subtree store failure")) + + settleTipRepair(t, bv, func() bool { return !bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped }) + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.True(t, b.stamped, "the re-mine is still owed") + require.Nil(t, b.pending) +} + +// TestMinedOnChainTracker_OnChainWriteClearsStamped: a completed write of the block as on +// the longest chain cleared every one of its txs, so no remine is owed. A write as off the +// chain cleared none. +func TestMinedOnChainTracker_OnChainWriteClearsStamped(t *testing.T) { + tracker := newMinedOnChainTracker() + hash := chainhash.HashH([]byte("b")) + + tracker.track(hash, 1, 1) + tracker.update(hash, tracker.snapshot()[hash].gen, func(e *minedOnChainBlock) { e.stamped = true }) + + require.True(t, tracker.markWritten(hash, false)) + require.True(t, tracker.snapshot()[hash].stamped, "an off-chain write leaves the remine owed") + + require.True(t, tracker.markWritten(hash, true)) + require.False(t, tracker.snapshot()[hash].stamped, "an on-chain write did the remine's work") +} + +// emptyIDsClient returns no ids from GetBlockHeaderIDs. +type emptyIDsClient struct { + blockchain.ClientI +} + +func (c *emptyIDsClient) GetBlockHeaderIDs(context.Context, *chainhash.Hash, uint64) ([]uint32, error) { + return []uint32{}, nil +} + +// TestMainChainIDs_EmptyIsAnError: no ids for the best block is a failed read, not a chain +// with no blocks on it; taken as one, every tracked block would look off the chain. +func TestMainChainIDs_EmptyIsAnError(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + _, _, err := bv.mainChainIDsLocked(f.ctx, &emptyIDsClient{ClientI: f.blockchainClient}) + require.Error(t, err) + require.Nil(t, bv.tipRepairMainChain, "nothing is cached") +} + +// TestRepairOvertakenTips_OnChainClearsDeferred: a block deferred after its job kept failing +// that is on the chain with no remine owed is not deferred any more, so if it leaves the +// chain its repair does not wait behind every other block. +func TestRepairOvertakenTips_OnChainClearsDeferred(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + hash := *f.blockB.Hash() + bv.minedOnChain.update(hash, bv.minedOnChain.snapshot()[hash].gen, func(e *minedOnChainBlock) { e.deferred = true }) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.False(t, bv.minedOnChain.snapshot()[hash].deferred) +} diff --git a/stores/blockchain/mock_last_n_blocks_test.go b/stores/blockchain/mock_last_n_blocks_test.go index 171b7a76f0..b383da7304 100644 --- a/stores/blockchain/mock_last_n_blocks_test.go +++ b/stores/blockchain/mock_last_n_blocks_test.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/require" ) -// TestMockStore_GetLastNBlocksNonPositiveN: a count of zero or less returns no blocks, as -// LIMIT does in the SQL store, instead of panicking on a negative slice capacity. +// TestMockStore_GetLastNBlocksNonPositiveN: a count of zero or less returns no blocks +// instead of panicking on a negative slice capacity. func TestMockStore_GetLastNBlocksNonPositiveN(t *testing.T) { m := NewMockStore() diff --git a/stores/utxo/sql/parent_outputs.go b/stores/utxo/sql/parent_outputs.go index 7ae24b58f7..3982be517c 100644 --- a/stores/utxo/sql/parent_outputs.go +++ b/stores/utxo/sql/parent_outputs.go @@ -2,7 +2,6 @@ package sql import ( "context" - "fmt" "github.com/bsv-blockchain/go-bt/v2/bscript" "github.com/bsv-blockchain/go-bt/v2/chainhash" @@ -110,7 +109,7 @@ const parentOutputsReadRetries = 3 func (s *Store) parentOutputsChunkWithRetry(ctx context.Context, chunk []outpointPair) (map[utxo.Outpoint]utxo.ParentOutput, error) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryOnLockError(ctx, s.logger, parentOutputsReadRetries, fmt.Sprintf("[ParentOutputsForValidation] chunk of %d outpoints", len(chunk)), func() error { + err := retryOnLockError(ctx, s.logger, parentOutputsReadRetries, "[ParentOutputsForValidation]", len(chunk), func() error { var err error found, err = s.parentOutputsChunk(ctx, chunk) diff --git a/stores/utxo/sql/parent_outputs_test.go b/stores/utxo/sql/parent_outputs_test.go index 7f8a89ba38..88d8a4ee79 100644 --- a/stores/utxo/sql/parent_outputs_test.go +++ b/stores/utxo/sql/parent_outputs_test.go @@ -82,7 +82,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { attempts++ if attempts < parentOutputsReadRetries { return lockErr @@ -106,7 +106,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { attempts++ return lockErr }) @@ -118,7 +118,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewStorageError("no such table: outputs") - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { attempts++ return other }) @@ -130,7 +130,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryOnLockError(cctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { attempts++ cancel() diff --git a/stores/utxo/sql/setmined_lock_retry_test.go b/stores/utxo/sql/setmined_lock_retry_test.go index 70f698bdb4..07ea39f61d 100644 --- a/stores/utxo/sql/setmined_lock_retry_test.go +++ b/stores/utxo/sql/setmined_lock_retry_test.go @@ -38,7 +38,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { var blockIDs map[chainhash.Hash][]uint32 - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { attempts++ if attempts < setMinedMultiRetries { return lockErr @@ -58,7 +58,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { attempts++ return lockErr }) @@ -70,7 +70,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewTxNotFoundError("transaction not found") - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { attempts++ return other }) @@ -82,7 +82,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryOnLockError(cctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { attempts++ cancel() diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 007de6632f..5fccb8d067 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -2421,18 +2421,19 @@ func isDeadlock(err error) bool { strings.Contains(msg, "(SQLSTATE "+usql.PgErrDeadlockDetected+")") } -// retryOnLockError runs op up to attempts times while it fails with an error -// isDeadlock recognises, and returns its last error. what names the operation -// in the warning logged before each retry. op must be safe to run again after -// such a failure: a failed statement or rolled-back transaction holds no locks. -func retryOnLockError(ctx context.Context, logger ulogger.Logger, attempts int, what string, op func() error) error { +// retryOnLockError runs op, over a chunk of items, up to attempts times while it +// fails with an error isDeadlock recognises, and returns its last error. what +// names the operation in the warning logged before each retry. op must be safe +// to run again after such a failure: a failed statement or rolled-back +// transaction holds no locks. +func retryOnLockError(ctx context.Context, logger ulogger.Logger, attempts int, what string, items int, op func() error) error { for attempt := 1; ; attempt++ { err := op() if err == nil || !isDeadlock(err) || attempt >= attempts { return err } - logger.Warnf("%s: lock error (attempt %d/%d), retrying: %v", what, attempt, attempts, err) + logger.Warnf("%s lock error (attempt %d/%d), retrying chunk of %d: %v", what, attempt, attempts, items, err) select { case <-ctx.Done(): @@ -3657,7 +3658,7 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min var chunkResult map[chainhash.Hash][]uint32 - err := retryOnLockError(ctx, s.logger, setMinedMultiRetries, fmt.Sprintf("[SetMinedMulti] chunk of %d txs", end-i), func() error { + err := retryOnLockError(ctx, s.logger, setMinedMultiRetries, "[SetMinedMulti]", end-i, func() error { var err error chunkResult, err = s.setMinedMultiChunk(ctx, hashes[i:end], minedBlockInfo) From 268b5fd973c02323d0eaaf3034eefa7749453d31 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 12:12:06 +0100 Subject: [PATCH 23/33] fix(blockvalidation): track the main chain in the same read that caches it, let a remine set a subtree aside and stay stamped A reorg was found by comparing the last best block with the new main-chain set, after that set was cached, so a failed header read on the slice that saw the reorg lost it for good. A new chain is now read once with GetBlockHeaders, which gives the ids and the blocks to track, and only cached after they are tracked. A remine stopped at a subtree it could not read and never re-mined the ones after it, starting again from the first each time. It now sets the subtree aside like an unmark job does and finishes the rest, but leaves the block stamped and backs off before the next remine, which the idle wake covers. - A write as off the chain clears no tx, so markWritten no longer reopens a finished or backed-off repair for it; an unset-mined write is not an on-chain write. - A pass that only sets subtrees aside warns once, on its first pass. - No ids for the best block is reported without a nil wrapped error. - retryOnLockError takes no item count. --- services/blockvalidation/BlockValidation.go | 3 +- .../blockvalidation/overtaken_tip_repair.go | 153 +++++++++++------- .../settxmined_overtaken_test.go | 110 ++++++++++--- stores/utxo/sql/parent_outputs.go | 2 +- stores/utxo/sql/parent_outputs_test.go | 8 +- stores/utxo/sql/setmined_lock_retry_test.go | 8 +- stores/utxo/sql/sql.go | 15 +- 7 files changed, 203 insertions(+), 96 deletions(-) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 73cc3b85db..5ba96465e8 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -332,7 +332,6 @@ type BlockValidation struct { // tipRepairMu), so a best block that only extends the chain costs one call. tipRepairMainChain map[uint32]bool tipRepairBestHash chainhash.Hash - tipRepairBestID uint32 // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. tipRepairRetry atomic.Bool // tipRepairWake requests a slice when the earliest pass over subtrees set aside comes @@ -1732,7 +1731,7 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // The write is complete, whichever onLongestChain this attempt used. Ask for a slice: a // chain change that happened while the write was in flight would otherwise wait for the // next one. - if u.minedOnChain.markWritten(*blockHash, onLongestChain) { + if u.minedOnChain.markWritten(*blockHash, onLongestChain && (len(unsetMined) == 0 || !unsetMined[0])) { u.signalTipRepair() } diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 32d15b39e4..6bb5ca1b46 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -147,7 +147,8 @@ func newMinedOnChainTracker() *minedOnChainTracker { return &minedOnChainTracker{blocks: make(map[chainhash.Hash]minedOnChainBlock), now: time.Now} } -// nextPendingPass returns the earliest nextPass of the blocks with subtrees set aside. +// nextPendingPass returns the earliest nextPass of the blocks with subtrees set aside, by +// an unmark job or a remine. func (t *minedOnChainTracker) nextPendingPass() (time.Time, bool) { if t == nil { return time.Time{}, false @@ -159,7 +160,8 @@ func (t *minedOnChainTracker) nextPendingPass() (time.Time, bool) { var earliest time.Time for _, b := range t.blocks { - if b.pending != nil && !b.done && (earliest.IsZero() || b.nextPass.Before(earliest)) { + owed := (b.pending != nil && !b.done) || (b.stamped && b.passes > 0) + if owed && (earliest.IsZero() || b.nextPass.Before(earliest)) { earliest = b.nextPass } } @@ -202,8 +204,9 @@ func (t *minedOnChainTracker) seed(hash chainhash.Hash, id, height uint32) { // markWritten records that a setTxMined write of the block has completed and reports // whether the block is tracked, in which case the caller asks for a repair slice. The write -// may have cleared txs after a repair read them, so a finished repair runs again. A write as -// on the longest chain cleared every tx of the block, so no remine is owed after it. +// as on the longest chain may have cleared txs after a repair read them, so a finished repair +// runs again, and it cleared every tx of the block, so no remine is owed after it. A write +// as off the chain clears none, and changes nothing here. func (t *minedOnChainTracker) markWritten(hash chainhash.Hash, onLongestChain bool) bool { if t == nil { return false @@ -217,15 +220,13 @@ func (t *minedOnChainTracker) markWritten(hash chainhash.Hash, onLongestChain bo return false } - b.done = false - b.clearPending() - if onLongestChain { + b.done = false + b.clearPending() b.stamped = false + t.blocks[hash] = b } - t.blocks[hash] = b - return true } @@ -458,27 +459,11 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b } } - hadMainChain, prevBestID := u.tipRepairMainChain != nil, u.tipRepairBestID - onMainChain, bestHeight, err := u.mainChainIDsLocked(ctx, blockchainClient) if err != nil { return false, err } - // The last best block left the chain. An unmark job leaves a tx mined while another block - // holding it is on the chain, so every block on the chain must be tracked, to have its - // own repair when it leaves. Blocks written on the chain by setTxMined are; one that - // reached the chain only through a child, its own setTxMined having run while it was off - // the chain, is tracked here. - if hadMainChain && !onMainChain[prevBestID] { - headers, metas, err := blockchainClient.GetBlockHeaders(ctx, &u.tipRepairBestHash, tipRepairWindow) - if err != nil { - return false, errors.NewServiceError("[repairOvertakenTips] failed to get main chain headers after a reorg", err) - } - - u.trackMinedHeaders(headers, metas) - } - tracked := u.minedOnChain.snapshot() for hash, b := range tracked { @@ -564,13 +549,13 @@ func (u *BlockValidation) startNextJob(ctx context.Context, blockchainClient blo onMainChain map[uint32]bool, bestHeight uint32, deferred bool) (more, started bool, err error) { // A re-marked block back on the chain first: its txs look unmined while mined. for hash, b := range tracked { - if b.deferred == deferred && b.stamped && onMainChain[b.id] { + if b.deferred == deferred && b.stamped && onMainChain[b.id] && !b.waiting(u.minedOnChain.now(), bestHeight) { return true, true, u.startJob(ctx, blockchainClient, remineJob, hash, b, bestHeight) } } for hash, b := range tracked { - if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && u.minedOnChain.now().Before(b.nextPass) && bestHeight <= b.passHeight) { + if b.deferred != deferred || b.done || onMainChain[b.id] || (b.pending != nil && b.waiting(u.minedOnChain.now(), bestHeight)) { continue } @@ -614,34 +599,49 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie case u.tipRepairMainChain != nil && bestHeader.HashPrevBlock.IsEqual(&u.tipRepairBestHash): // GetBestBlockHeader's meta carries no block id over gRPC, so ask for it. ids, err := blockchainClient.GetBlockHeaderIDs(ctx, &bestHash, 1) - if err != nil || len(ids) == 0 { + if err != nil { return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get best block id", err) } + if len(ids) == 0 { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] no id for best block %s", bestHash.String()) + } + // Ids that fell out of the window stay in the set; tipRepairExpired drops their // blocks before the set is consulted for them. u.tipRepairMainChain[ids[0]] = true u.tipRepairBestHash = bestHash - u.tipRepairBestID = ids[0] - return u.tipRepairMainChain, bestMeta.Height, nil } - // No ids is a failed read, not a chain with no blocks: taken as one, every tracked block - // would look off the chain. - ids, err := blockchainClient.GetBlockHeaderIDs(ctx, &bestHash, tipRepairWindow) - if err != nil || len(ids) == 0 { - return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get main chain block ids", err) + // A new chain, not an extension of the one seen last time: a reorg, or several blocks at + // once. An unmark job leaves a tx mined while another block holding it is on the chain, + // so every block on the chain must be tracked, to have its own repair when it leaves. + // Blocks written on the chain by setTxMined are; one that reached the chain only through + // a child, its own setTxMined having run while it was off the chain, is tracked here, + // before the set is cached, so a failed read is retried in full. + headers, metas, err := blockchainClient.GetBlockHeaders(ctx, &bestHash, tipRepairWindow) + if err != nil { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get main chain headers", err) + } + + // No headers is a failed read, not a chain with no blocks: taken as one, every tracked + // block would look off the chain. + if len(metas) == 0 { + return nil, 0, errors.NewServiceError("[repairOvertakenTips] no main chain headers for best block %s", bestHash.String()) } - onMainChain := make(map[uint32]bool, len(ids)) - for _, id := range ids { - onMainChain[id] = true + u.trackMinedHeaders(headers, metas) + + onMainChain := make(map[uint32]bool, len(metas)) + for _, meta := range metas { + if meta != nil { + onMainChain[meta.ID] = true + } } u.tipRepairMainChain = onMainChain u.tipRepairBestHash = bestHash - u.tipRepairBestID = ids[0] return onMainChain, bestMeta.Height, nil } @@ -656,9 +656,14 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch u.minedOnChain.update(hash, b.gen, func(e *minedOnChainBlock) { e.deferred = false + // A remine reads the whole block. The unmark job's set-aside subtrees no longer + // apply; the remine's own back-off, from an earlier remine, stays. if kind == remineJob { e.done = false - e.clearPending() + + if e.pending != nil { + e.clearPending() + } } }) @@ -674,14 +679,13 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone -// is skipped with an error log. For an unmark job, one that still cannot be loaded after -// tipRepairMaxAttempts failed loads, whatever the error, is set aside, so the rest of the -// block is repaired: a run of short reads looks the same as a corrupt file, and a store that -// stays down must not hold up the later subtrees. The block's next unmark job reads the -// set-aside subtrees again (see finishJob). A remine job never sets one aside: finishing -// would drop the re-mine of its txs, so its load errors fail the step and the job is -// retried, deferred behind other blocks once it keeps failing. Until a subtree is set aside, -// a load error fails the step, to be retried. +// is skipped with an error log. One that still cannot be loaded after tipRepairMaxAttempts +// failed loads, whatever the error, is set aside, so the rest of the block is repaired: a +// run of short reads looks the same as a corrupt file, and a store that stays down must not +// hold up the later subtrees. The block's next unmark job reads the set-aside subtrees +// again; a remine job with subtrees set aside leaves the block stamped, so a later remine +// does them (see finishJob). Until a subtree is set aside, a load error fails the step, to +// be retried. func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockchain.ClientI, job *tipRepairJob, onMainChain map[uint32]bool) error { for len(job.current) == 0 { if job.nextSubtree >= len(job.subtrees) { @@ -698,7 +702,7 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha switch { case isPermanentRepairError(err): u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) - case job.kind == unmarkJob && ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: + case ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) job.unreadable = append(job.unreadable, subtreeHash) default: @@ -779,6 +783,7 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { // A block with subtrees set aside is not done: its next job reads them again, // after the other blocks have had their turn, and once twice as long as last time // has passed or a new block has arrived, whichever comes first. + passes := 0 u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { e.done = len(job.unreadable) == 0 e.pending = job.unreadable @@ -786,30 +791,60 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { e.failures = 0 if !e.done { - e.passes++ - e.nextPass = u.minedOnChain.now().Add(min(tipRepairPendingBaseBackoff< 0 || len(job.unreadable) > 0 { + switch { + case job.marked > 0: u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain after its txs were marked mined, marked %d of %d txs as not on longest chain", job.hash.String(), job.marked, job.total) - } else { - u.logger.Infof("[repairOvertakenTips][%s] block is off the longest chain, none of its %d txs needed marking as not on longest chain", job.hash.String(), job.total) + case len(job.unreadable) > 0 && passes == 1: + u.logger.Warnf("[repairOvertakenTips][%s] block left the longest chain, %d of its subtrees could not be read and are set aside, their txs may look mined until a later pass reads them", job.hash.String(), len(job.unreadable)) + default: + // Such as a fork block the restart seed tracks that was never on the chain, or a + // later pass over subtrees that still cannot be read. + u.logger.Infof("[repairOvertakenTips][%s] block is off the longest chain, marked %d of %d txs as not on longest chain, %d subtrees set aside", job.hash.String(), job.marked, job.total, len(job.unreadable)) } case remineJob: + // A remine that set subtrees aside leaves the block stamped: their re-mine is still + // owed, and a later remine of the block does it, backed off like an unmark pass. u.minedOnChain.update(job.hash, job.gen, func(e *minedOnChainBlock) { - e.stamped = false e.failures = 0 + + if len(job.unreadable) > 0 { + e.deferred = true + u.backOffPass(e, job) + + return + } + + e.stamped = false + e.passes = 0 + e.nextPass = time.Time{} + e.passHeight = 0 }) - u.logger.Infof("[repairOvertakenTips][%s] block is back on the longest chain, marked %d of %d txs mined again", job.hash.String(), job.marked, job.total) + u.logger.Infof("[repairOvertakenTips][%s] block is back on the longest chain, marked %d of %d txs mined again, %d subtrees set aside", job.hash.String(), job.marked, job.total, len(job.unreadable)) } } +// backOffPass records a pass that ended with subtrees set aside: the next is due twice as +// long after it as the last one, up to tipRepairPendingMaxBackoff, or once a block arrives +// after it. +func (u *BlockValidation) backOffPass(e *minedOnChainBlock, job *tipRepairJob) { + e.passes++ + e.nextPass = u.minedOnChain.now().Add(min(tipRepairPendingBaseBackoff< 0 && now.Before(b.nextPass) && bestHeight <= b.passHeight +} + // repairFailed records a failed step for a block and returns err. A permanent failure (the // block or its subtrees are gone) counts toward tipRepairMaxAttempts, after which the block // is dropped; a transient one is only retried. diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 71d19f75ed..891db356ec 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -973,7 +973,7 @@ func (c *seedFailClient) GetBlockHeaderIDs(ctx context.Context, hash *chainhash. // TestRepairOvertakenTips_SeedFailures: a blockchain call failing during the restart seed or // the main-chain read fails the slice and leaves the seed to run again. func TestRepairOvertakenTips_SeedFailures(t *testing.T) { - for _, failOn := range []string{"GetBestBlockHeader", "GetBlockHeaders", "GetLastNBlocks", "GetBlockHeader", "GetBlockHeaderIDs"} { + for _, failOn := range []string{"GetBestBlockHeader", "GetBlockHeaders", "GetLastNBlocks", "GetBlockHeader"} { t.Run(failOn, func(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) @@ -1894,12 +1894,14 @@ func TestRepairOvertakenTips_BackOffFromPassEnd(t *testing.T) { require.Equal(t, left, store.short.Load(), "no new pass before a block arrives after the last one ended") } -// TestRepairOvertakenTips_RemineNeverSetsAside: a remine job whose subtree keeps failing to -// load must not set it aside and finish: that drops the re-mine of its txs for good. The -// block stays stamped and the remine is retried. -func TestRepairOvertakenTips_RemineNeverSetsAside(t *testing.T) { - f := newOvertakeFixture(t) +// TestRepairOvertakenTips_RemineSetsAsideKeepsStamped: a remine job whose subtree keeps +// failing to load sets it aside and re-mines the later subtrees, but the block stays +// stamped: the set-aside subtree's re-mine is still owed, and a later pass, once its reads +// recover, does it. +func TestRepairOvertakenTips_RemineSetsAsideKeepsStamped(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) bv := f.newBlockValidation(t) + clock := f.fakeTipRepairClock(bv) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) _ = f.overtakeB(t) @@ -1907,16 +1909,88 @@ func TestRepairOvertakenTips_RemineNeverSetsAside(t *testing.T) { _, err := drainTipRepair(t, bv) require.NoError(t, err) require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped, "precondition: B's txs were re-marked") + require.NotZero(t, f.unminedSince(t, f.bOnly), "precondition") + require.NotZero(t, f.unminedSince(t, f.later), "precondition") f.extendB(t) - bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewStorageError("injected subtree store failure")) + store := f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewStorageError("injected subtree store failure")) + bv.subtreeStore = store - settleTipRepair(t, bv, func() bool { return !bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped }) + for i := 0; i <= tipRepairMaxAttempts; i++ { + _, _ = drainTipRepair(t, bv) + } - b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] - require.True(t, b.stamped, "the re-mine is still owed") - require.Nil(t, b.pending) + require.Zero(t, f.unminedSince(t, f.later), "the later subtree is re-mined past the unreadable one") + require.NotZero(t, f.unminedSince(t, f.bOnly), "precondition: subtree 0 is not re-mined yet") + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped, "the re-mine of subtree 0 is still owed") + + store.fail.Store(0) + clock.advance(tipRepairPendingMaxBackoff) + + _, err = drainTipRepair(t, bv) + require.NoError(t, err) + require.Zero(t, f.unminedSince(t, f.bOnly), "subtree 0 is re-mined once it reads") + require.False(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped) +} + +// failHeadersOnceClient fails the next GetBlockHeaders call once armed. +type failHeadersOnceClient struct { + blockchain.ClientI + fail atomic.Bool +} + +func (c *failHeadersOnceClient) GetBlockHeaders(ctx context.Context, hash *chainhash.Hash, n uint64) ([]*model.BlockHeader, []*model.BlockHeaderMeta, error) { + if c.fail.Swap(false) { + return nil, nil, errors.NewServiceError("injected GetBlockHeaders failure") + } + + return c.ClientI.GetBlockHeaders(ctx, hash, n) +} + +// TestRepairOvertakenTips_ReorgTrackingSurvivesFailedRead: a failed read on the slice that +// first sees the reorg must not lose it: the next slice still tracks B', which reached the +// chain through its child. +func TestRepairOvertakenTips_ReorgTrackingSurvivesFailedRead(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + client := &failHeadersOnceClient{ClientI: f.blockchainClient} + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _, err := bv.tipRepairSlice(f.ctx, client) + require.NoError(t, err) + + blockC2 := f.overtakeB(t) + + client.fail.Store(true) + _, err = bv.tipRepairSlice(f.ctx, client) + require.Error(t, err, "precondition: the slice that sees the reorg fails") + + _, err = bv.tipRepairSlice(f.ctx, client) + require.NoError(t, err) + + _, tracked := bv.minedOnChain.snapshot()[*blockC2.Header.HashPrevBlock] + require.True(t, tracked, "B' is tracked on the retried slice") +} + +// TestMinedOnChainTracker_OffChainWriteKeepsDone: a write as off the longest chain clears no +// tx, so a finished or backed-off repair is not opened again by it. +func TestMinedOnChainTracker_OffChainWriteKeepsDone(t *testing.T) { + tracker := newMinedOnChainTracker() + hash := chainhash.HashH([]byte("b")) + subtree := chainhash.HashH([]byte("s")) + + tracker.track(hash, 1, 1) + tracker.update(hash, tracker.snapshot()[hash].gen, func(e *minedOnChainBlock) { + e.done = true + e.pending = []*chainhash.Hash{&subtree} + }) + + require.True(t, tracker.markWritten(hash, false)) + + b := tracker.snapshot()[hash] + require.True(t, b.done) + require.NotNil(t, b.pending) } // TestMinedOnChainTracker_OnChainWriteClearsStamped: a completed write of the block as on @@ -1936,22 +2010,22 @@ func TestMinedOnChainTracker_OnChainWriteClearsStamped(t *testing.T) { require.False(t, tracker.snapshot()[hash].stamped, "an on-chain write did the remine's work") } -// emptyIDsClient returns no ids from GetBlockHeaderIDs. -type emptyIDsClient struct { +// emptyHeadersClient returns no headers from GetBlockHeaders. +type emptyHeadersClient struct { blockchain.ClientI } -func (c *emptyIDsClient) GetBlockHeaderIDs(context.Context, *chainhash.Hash, uint64) ([]uint32, error) { - return []uint32{}, nil +func (c *emptyHeadersClient) GetBlockHeaders(context.Context, *chainhash.Hash, uint64) ([]*model.BlockHeader, []*model.BlockHeaderMeta, error) { + return []*model.BlockHeader{}, []*model.BlockHeaderMeta{}, nil } -// TestMainChainIDs_EmptyIsAnError: no ids for the best block is a failed read, not a chain -// with no blocks on it; taken as one, every tracked block would look off the chain. +// TestMainChainIDs_EmptyIsAnError: no headers for the best block is a failed read, not a +// chain with no blocks on it; taken as one, every tracked block would look off the chain. func TestMainChainIDs_EmptyIsAnError(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) - _, _, err := bv.mainChainIDsLocked(f.ctx, &emptyIDsClient{ClientI: f.blockchainClient}) + _, _, err := bv.mainChainIDsLocked(f.ctx, &emptyHeadersClient{ClientI: f.blockchainClient}) require.Error(t, err) require.Nil(t, bv.tipRepairMainChain, "nothing is cached") } diff --git a/stores/utxo/sql/parent_outputs.go b/stores/utxo/sql/parent_outputs.go index 3982be517c..9c704b5df5 100644 --- a/stores/utxo/sql/parent_outputs.go +++ b/stores/utxo/sql/parent_outputs.go @@ -109,7 +109,7 @@ const parentOutputsReadRetries = 3 func (s *Store) parentOutputsChunkWithRetry(ctx context.Context, chunk []outpointPair) (map[utxo.Outpoint]utxo.ParentOutput, error) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryOnLockError(ctx, s.logger, parentOutputsReadRetries, "[ParentOutputsForValidation]", len(chunk), func() error { + err := retryOnLockError(ctx, s.logger, parentOutputsReadRetries, "[ParentOutputsForValidation] outpoints chunk", func() error { var err error found, err = s.parentOutputsChunk(ctx, chunk) diff --git a/stores/utxo/sql/parent_outputs_test.go b/stores/utxo/sql/parent_outputs_test.go index 88d8a4ee79..7f8a89ba38 100644 --- a/stores/utxo/sql/parent_outputs_test.go +++ b/stores/utxo/sql/parent_outputs_test.go @@ -82,7 +82,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { var found map[utxo.Outpoint]utxo.ParentOutput - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ if attempts < parentOutputsReadRetries { return lockErr @@ -106,7 +106,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ return lockErr }) @@ -118,7 +118,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewStorageError("no such table: outputs") - err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ return other }) @@ -130,7 +130,7 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryOnLockError(cctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", 1, func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, parentOutputsReadRetries, "[ParentOutputsForValidation]", func() error { attempts++ cancel() diff --git a/stores/utxo/sql/setmined_lock_retry_test.go b/stores/utxo/sql/setmined_lock_retry_test.go index 07ea39f61d..70f698bdb4 100644 --- a/stores/utxo/sql/setmined_lock_retry_test.go +++ b/stores/utxo/sql/setmined_lock_retry_test.go @@ -38,7 +38,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { var blockIDs map[chainhash.Hash][]uint32 - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ if attempts < setMinedMultiRetries { return lockErr @@ -58,7 +58,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { t.Run("a lock that never clears returns the lock error", func(t *testing.T) { attempts := 0 - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ return lockErr }) @@ -70,7 +70,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { attempts := 0 other := errors.NewTxNotFoundError("transaction not found") - err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { + err := retryOnLockError(ctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ return other }) @@ -82,7 +82,7 @@ func TestSetMinedMultiRetriesLockError(t *testing.T) { cctx, cancel := context.WithCancel(ctx) attempts := 0 - err := retryOnLockError(cctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", 1, func() error { + err := retryOnLockError(cctx, ulogger.TestLogger{}, setMinedMultiRetries, "[SetMinedMulti]", func() error { attempts++ cancel() diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 5fccb8d067..98e609e564 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -2421,19 +2421,18 @@ func isDeadlock(err error) bool { strings.Contains(msg, "(SQLSTATE "+usql.PgErrDeadlockDetected+")") } -// retryOnLockError runs op, over a chunk of items, up to attempts times while it -// fails with an error isDeadlock recognises, and returns its last error. what -// names the operation in the warning logged before each retry. op must be safe -// to run again after such a failure: a failed statement or rolled-back -// transaction holds no locks. -func retryOnLockError(ctx context.Context, logger ulogger.Logger, attempts int, what string, items int, op func() error) error { +// retryOnLockError runs op up to attempts times while it fails with an error +// isDeadlock recognises, and returns its last error. what names the operation +// in the warning logged before each retry. op must be safe to run again after +// such a failure: a failed statement or rolled-back transaction holds no locks. +func retryOnLockError(ctx context.Context, logger ulogger.Logger, attempts int, what string, op func() error) error { for attempt := 1; ; attempt++ { err := op() if err == nil || !isDeadlock(err) || attempt >= attempts { return err } - logger.Warnf("%s lock error (attempt %d/%d), retrying chunk of %d: %v", what, attempt, attempts, items, err) + logger.Warnf("%s lock error (attempt %d/%d), retrying: %v", what, attempt, attempts, err) select { case <-ctx.Done(): @@ -3658,7 +3657,7 @@ func (s *Store) SetMinedMulti(ctx context.Context, hashes []*chainhash.Hash, min var chunkResult map[chainhash.Hash][]uint32 - err := retryOnLockError(ctx, s.logger, setMinedMultiRetries, "[SetMinedMulti]", end-i, func() error { + err := retryOnLockError(ctx, s.logger, setMinedMultiRetries, "[SetMinedMulti] chunk", func() error { var err error chunkResult, err = s.setMinedMultiChunk(ctx, hashes[i:end], minedBlockInfo) From e4b5743a6bede3553677b12c43762216a274d612 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 12:21:59 +0100 Subject: [PATCH 24/33] fix(blockvalidation): end the back-off when an unmark pass finishes the block, wake only for a pass still to come An unmark pass that read the subtrees set aside and finished the block left passes and nextPass set, so nextPendingPass still reported a stamped block off the chain as owed a pass that no slice would start. The wake timer was then armed with a negative delay on every slice. A clean pass now resets the back-off, nextPendingPass counts only blocks that are not done, and the timer is armed only for a pass in the future. --- .../blockvalidation/overtaken_tip_repair.go | 21 ++++++++++----- .../settxmined_overtaken_test.go | 26 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 6bb5ca1b46..2d51ca3f3d 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -160,7 +160,8 @@ func (t *minedOnChainTracker) nextPendingPass() (time.Time, bool) { var earliest time.Time for _, b := range t.blocks { - owed := (b.pending != nil && !b.done) || (b.stamped && b.passes > 0) + // What startNextJob would start: a block that is done is owed nothing. + owed := !b.done && (b.pending != nil || (b.stamped && b.passes > 0)) if owed && (earliest.IsZero() || b.nextPass.Before(earliest)) { earliest = b.nextPass } @@ -354,12 +355,16 @@ func (u *BlockValidation) runTipRepairSlice(ctx context.Context, blockchainClien // Nothing is due now. A pass over subtrees set aside comes due at its nextPass even on // an idle chain, with no block or write to request a slice, so wake for the earliest. + // A pass already due that this slice did not start is not waiting on the clock, and + // waking for it at once would only repeat this slice. if at, ok := u.minedOnChain.nextPendingPass(); ok { - if u.tipRepairWake != nil { - u.tipRepairWake.Stop() - } + if wait := at.Sub(u.minedOnChain.now()); wait > 0 { + if u.tipRepairWake != nil { + u.tipRepairWake.Stop() + } - u.tipRepairWake = time.AfterFunc(at.Sub(u.minedOnChain.now()), u.signalTipRepair) + u.tipRepairWake = time.AfterFunc(wait, u.signalTipRepair) + } } } @@ -790,7 +795,11 @@ func (u *BlockValidation) finishJob(job *tipRepairJob) { e.deferred = !e.done e.failures = 0 - if !e.done { + if e.done { + e.passes = 0 + e.nextPass = time.Time{} + e.passHeight = 0 + } else { u.backOffPass(e, job) } diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/settxmined_overtaken_test.go index 891db356ec..87eac9de23 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/settxmined_overtaken_test.go @@ -2046,3 +2046,29 @@ func TestRepairOvertakenTips_OnChainClearsDeferred(t *testing.T) { require.NoError(t, err) require.False(t, bv.minedOnChain.snapshot()[hash].deferred) } + +// TestRepairOvertakenTips_CleanPassEndsBackOff: once a pass reads the subtrees set aside and +// the block is done, nothing is owed: no pending pass may be reported, or the worker's wake +// timer fires at once on every slice while the block stays off the chain. +func TestRepairOvertakenTips_CleanPassEndsBackOff(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + clock := f.fakeTipRepairClock(bv) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, tipRepairSubtreeReadAttempts*tipRepairMaxAttempts) + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].stamped, "precondition: the first pass re-marked txs") + + clock.advance(tipRepairPendingBaseBackoff) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "precondition: the clean pass finished the block") + + _, ok := bv.minedOnChain.nextPendingPass() + require.False(t, ok, "no pass is owed by a done block off the chain") +} From 373f16de1ad34f1f062ad6df1a624c0b80a00338 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 14:57:57 +0100 Subject: [PATCH 25/33] fix(blockvalidation): finish the due overtaken-tip repair before setting mined_set, read headers within the configured caps The pruner processes a height only once its block has mined_set, and the repair relied on winning a race against it. setTxMinedStatus now runs every repair the chain has made due before SetBlockMinedSet, so a tx that looks mined off the chain is re-marked (with its unmined_since backdated, so the pruner's next phase 1 preserves its parents) before the pruner can process that height. A pass over set-aside subtrees is due on every block, so a subtree whose reads have recovered is read there too. A repair that keeps failing leaves mined_set unset, to be retried. A subtree that still cannot be read cannot be named, so it stays set aside and is counted in the new teranode_blockvalidation_tip_repair_subtrees_set_aside gauge. The seed and the main-chain read asked for 1000 headers and 1000 recent blocks in one call. The blockchain service rejects requests above blockchain_maxBlockHeadersPerRequest and blockchain_maxBlocksByHeightRange, which may be set lower, so both now read in pages within those caps. The overtaken-tip tests move to overtaken_tip_repair_test.go and the mock's to mock_test.go, after the code they test. --- services/blockvalidation/BlockValidation.go | 7 + services/blockvalidation/metrics.go | 16 +- .../blockvalidation/overtaken_tip_repair.go | 164 +++++++++++++++--- ...n_test.go => overtaken_tip_repair_test.go} | 153 ++++++++++++++++ ...ock_last_n_blocks_test.go => mock_test.go} | 0 5 files changed, 314 insertions(+), 26 deletions(-) rename services/blockvalidation/{settxmined_overtaken_test.go => overtaken_tip_repair_test.go} (92%) rename stores/blockchain/{mock_last_n_blocks_test.go => mock_test.go} (100%) diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 2655759ff9..0ce38e0efe 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1808,6 +1808,13 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain u.logger.Warnf("[setTxMined][%s] failed closing subtrees after setting mined: %v", block.Hash().String(), releaseErr) } + // The pruner processes a height only once its block has mined_set. Every repair this + // block's chain makes due runs first, so no parent of a tx that still looks mined off the + // chain is pruned before the tx is re-marked and its parents preserved. + if err = u.drainTipRepair(ctx, u.blockchainClient); err != nil { + return errors.NewProcessingError("[setTxMined][%s] overtaken-tip repair failed, mined_set not set", block.Hash().String(), err) + } + // update block mined_set to true if err = u.blockchainClient.SetBlockMinedSet(ctx, blockHash); err != nil { return errors.NewServiceError("[setTxMined][%s] failed to set block mined", block.Hash().String(), err) diff --git a/services/blockvalidation/metrics.go b/services/blockvalidation/metrics.go index cd888f08ac..503556beae 100644 --- a/services/blockvalidation/metrics.go +++ b/services/blockvalidation/metrics.go @@ -89,9 +89,10 @@ var ( // unbounded Prometheus cardinality (one permanent series per distinct hash // over the node's lifetime). The specific block hash is recorded in the // accompanying log lines for manual repair. - prometheusBlockValidationSetMinedRetries prometheus.Counter - prometheusBlockValidationSetMinedDrops prometheus.Counter - prometheusBlockValidationSetMinedEnqueueOverflow prometheus.Counter + prometheusBlockValidationSetMinedRetries prometheus.Counter + prometheusBlockValidationSetMinedDrops prometheus.Counter + prometheusBlockValidationSetMinedEnqueueOverflow prometheus.Counter + prometheusBlockValidationTipRepairSubtreesSetAside prometheus.Gauge // outpoint-only fast-path counter: incremented once per block when the // below-checkpoint outpoint-only path is active (setting on, height ≤ highest @@ -278,6 +279,15 @@ func _initPrometheusMetrics() { }, ) + prometheusBlockValidationTipRepairSubtreesSetAside = promauto.NewGauge( + prometheus.GaugeOpts{ + Namespace: "teranode", + Subsystem: "blockvalidation", + Name: "tip_repair_subtrees_set_aside", + Help: "Subtrees of overtaken blocks the overtaken-tip repair could not read and set aside. Their txs may look mined while off the longest chain, and the pruner cannot protect those txs' parents, until a later pass reads them. Above zero for long means a subtree file is corrupt or its store is failing.", + }, + ) + prometheusBlockValidationQuickCommitAddBlock = promauto.NewHistogram( prometheus.HistogramOpts{ Namespace: "teranode", diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 2d51ca3f3d..e735ff9cf9 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -170,6 +170,23 @@ func (t *minedOnChainTracker) nextPendingPass() (time.Time, bool) { return earliest, !earliest.IsZero() } +// setAsideSubtrees returns how many subtrees of the tracked blocks an unmark job set aside. +func (t *minedOnChainTracker) setAsideSubtrees() int { + if t == nil { + return 0 + } + + t.mu.Lock() + defer t.mu.Unlock() + + n := 0 + for _, b := range t.blocks { + n += len(b.pending) + } + + return n +} + // track watches a block whose on-chain write is starting: the write clears the block's txs // again, so the block gets a new gen and starts over. func (t *minedOnChainTracker) track(hash chainhash.Hash, id, height uint32) { @@ -319,6 +336,33 @@ type tipRepairJob struct { height uint32 } +// drainTipRepair runs repair slices until none is due. setTxMinedStatus calls it before +// setting a block's mined_set, which is what lets the pruner process that height: a repair +// the chain has made due is finished first, so its txs are re-marked, and their parents +// preserved by the pruner's next phase 1, before any parent can be pruned. A pass over +// subtrees set aside is due on every new block, so a subtree whose reads have recovered is +// read here too. One that still cannot be read stays set aside: its txs cannot be named, and +// tipRepairSubtreesSetAside reports it. A slice that keeps failing returns its error after +// tipRepairMaxAttempts in a row; the caller leaves mined_set unset, to be retried. +func (u *BlockValidation) drainTipRepair(ctx context.Context, blockchainClient blockchain.ClientI) error { + for failures := 0; ; { + more, err := u.tipRepairSlice(ctx, blockchainClient) + if err != nil { + if failures++; failures >= tipRepairMaxAttempts || ctx.Err() != nil { + return err + } + + continue + } + + failures = 0 + + if !more { + return nil + } + } +} + // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending // request covers any number of calls. func (u *BlockValidation) signalTipRepair() { @@ -377,44 +421,75 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo return errors.NewServiceError("[repairOvertakenTips] seed: failed to get best block header", err) } - headers, metas, err := blockchainClient.GetBlockHeaders(ctx, bestHeader.Hash(), tipRepairWindow) + headers, metas, err := u.mainChainHeaders(ctx, blockchainClient, *bestHeader.Hash()) if err != nil { return errors.NewServiceError("[repairOvertakenTips] seed: failed to get main chain headers", err) } u.trackMinedHeaders(headers, metas) - // Fork blocks near the tip: one query for the highest blocks, side chains included. - // GetChainTips would do too, but on SQLite it holds a row cursor open across nested - // queries and deadlocks against a concurrent write transaction. - recent, err := blockchainClient.GetLastNBlocks(ctx, tipRepairSeedMaxBlocks, true, 0) - if err != nil { - return errors.NewServiceError("[repairOvertakenTips] seed: failed to get recent blocks", err) + // Fork blocks near the tip: the highest blocks, side chains included, in pages within + // blockchain_maxBlocksByHeightRange. GetChainTips would do too, but on SQLite it holds a + // row cursor open across nested queries and deadlocks against a concurrent write + // transaction. + pageMax := int64(u.settings.BlockChain.MaxBlocksByHeightRange) + if pageMax <= 0 { + pageMax = tipRepairSeedMaxBlocks } - for _, info := range recent { - if info.Height+tipRepairSeedForkDepth < bestMeta.Height { - break // highest first - } + seen := make(map[chainhash.Hash]bool) + fromHeight := uint32(0) // 0 is no upper bound - header, err := model.NewBlockHeaderFromBytes(info.BlockHeader) + for read := int64(0); read < tipRepairSeedMaxBlocks; { + page := min(tipRepairSeedMaxBlocks-read, pageMax) + + recent, err := blockchainClient.GetLastNBlocks(ctx, page, true, fromHeight) if err != nil { - return errors.NewProcessingError("[repairOvertakenTips] seed: invalid block header at height %d", info.Height, err) + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get recent blocks", err) } - _, meta, err := blockchainClient.GetBlockHeader(ctx, header.Hash()) - if err != nil { - return errors.NewServiceError("[repairOvertakenTips] seed: failed to get block header %s", header.Hash().String(), err) + read += int64(len(recent)) + added := false + + for _, info := range recent { + if info.Height+tipRepairSeedForkDepth < bestMeta.Height { + u.tipRepairSeeded = true // highest first: the rest are deeper + return nil + } + + header, err := model.NewBlockHeaderFromBytes(info.BlockHeader) + if err != nil { + return errors.NewProcessingError("[repairOvertakenTips] seed: invalid block header at height %d", info.Height, err) + } + + // A page starts at the lowest height of the last one, which it may share. + if seen[*header.Hash()] { + continue + } + + seen[*header.Hash()] = true + added = true + + _, meta, err := blockchainClient.GetBlockHeader(ctx, header.Hash()) + if err != nil { + return errors.NewServiceError("[repairOvertakenTips] seed: failed to get block header %s", header.Hash().String(), err) + } + + if meta == nil || meta.Invalid { + continue + } + + // A block whose mined_set is not written is watched too: a setTxMined that failed + // part way before the restart may have cleared some of its txs, and its retry, + // with onLongestChain=false once it is off the chain, leaves unmined_since alone. + u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height) } - if meta == nil || meta.Invalid { - continue + if int64(len(recent)) < page || !added || recent[len(recent)-1].Height == 0 { + break } - // A block whose mined_set is not written is watched too: a setTxMined that failed - // part way before the restart may have cleared some of its txs, and its retry, - // with onLongestChain=false once it is off the chain, leaves unmined_since alone. - u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height) + fromHeight = recent[len(recent)-1].Height } u.tipRepairSeeded = true @@ -422,6 +497,45 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo return nil } +// mainChainHeaders returns the tipRepairWindow headers ending at best, best first, read in +// pages within blockchain_maxBlockHeadersPerRequest, which the blockchain service enforces +// and may be set below the window. +func (u *BlockValidation) mainChainHeaders(ctx context.Context, blockchainClient blockchain.ClientI, best chainhash.Hash) ([]*model.BlockHeader, []*model.BlockHeaderMeta, error) { + pageMax := uint64(tipRepairWindow) + if limit := u.settings.BlockChain.MaxBlockHeadersPerRequest; limit > 0 { + pageMax = min(pageMax, uint64(limit)) + } + + headers := make([]*model.BlockHeader, 0, tipRepairWindow) + metas := make([]*model.BlockHeaderMeta, 0, tipRepairWindow) + next := best + + for uint64(len(headers)) < tipRepairWindow { + page := min(tipRepairWindow-uint64(len(headers)), pageMax) + + pageHeaders, pageMetas, err := blockchainClient.GetBlockHeaders(ctx, &next, page) + if err != nil { + return nil, nil, err + } + + headers = append(headers, pageHeaders...) + metas = append(metas, pageMetas...) + + if uint64(len(pageHeaders)) < page { + break // reached genesis + } + + prev := pageHeaders[len(pageHeaders)-1].HashPrevBlock + if prev == nil || prev.IsEqual(&chainhash.Hash{}) { + break + } + + next = *prev + } + + return headers, metas, nil +} + func (u *BlockValidation) trackMinedHeaders(headers []*model.BlockHeader, metas []*model.BlockHeaderMeta) { for i, meta := range metas { if i >= len(headers) || meta == nil || meta.Invalid { @@ -450,6 +564,10 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b u.tipRepairMu.Lock() defer u.tipRepairMu.Unlock() + defer func() { + prometheusBlockValidationTipRepairSubtreesSetAside.Set(float64(u.minedOnChain.setAsideSubtrees())) + }() + // Nothing to do until this process has written a block as on the longest chain. The // restart seed waits for that too: after a restart the next block's setTxMined comes // within one block interval, and a process that never marks blocks mined (or a test @@ -625,7 +743,7 @@ func (u *BlockValidation) mainChainIDsLocked(ctx context.Context, blockchainClie // Blocks written on the chain by setTxMined are; one that reached the chain only through // a child, its own setTxMined having run while it was off the chain, is tracked here, // before the set is cached, so a failed read is retried in full. - headers, metas, err := blockchainClient.GetBlockHeaders(ctx, &bestHash, tipRepairWindow) + headers, metas, err := u.mainChainHeaders(ctx, blockchainClient, bestHash) if err != nil { return nil, 0, errors.NewServiceError("[repairOvertakenTips] failed to get main chain headers", err) } diff --git a/services/blockvalidation/settxmined_overtaken_test.go b/services/blockvalidation/overtaken_tip_repair_test.go similarity index 92% rename from services/blockvalidation/settxmined_overtaken_test.go rename to services/blockvalidation/overtaken_tip_repair_test.go index 87eac9de23..07908fbc74 100644 --- a/services/blockvalidation/settxmined_overtaken_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -30,6 +30,7 @@ import ( "github.com/bsv-blockchain/teranode/stores/utxo/fields" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util/test" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" ) @@ -2072,3 +2073,155 @@ func TestRepairOvertakenTips_CleanPassEndsBackOff(t *testing.T) { _, ok := bv.minedOnChain.nextPendingPass() require.False(t, ok, "no pass is owed by a done block off the chain") } + +// minedSetProbe runs at on each SetBlockMinedSet of block, before the write: the moment the +// pruner may start processing that height. calls counts the runs of at that have returned. +type minedSetProbe struct { + blockchain.ClientI + block chainhash.Hash + at func() + calls atomic.Int32 +} + +func (c *minedSetProbe) SetBlockMinedSet(ctx context.Context, blockHash *chainhash.Hash) error { + if *blockHash == c.block { + c.at() + c.calls.Add(1) + } + + return c.ClientI.SetBlockMinedSet(ctx, blockHash) +} + +// TestSetTxMinedStatus_RepairsOvertakenBlockBeforeMinedSet: the pruner processes a height +// only once its block has mined_set. So the setTxMined of the block that overtook B must +// finish B's repair before it sets mined_set, or the pruner can delete a parent of a tx of +// B that still looks mined. +func TestSetTxMinedStatus_RepairsOvertakenBlockBeforeMinedSet(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + var atMinedSet uint32 + + probe := &minedSetProbe{ClientI: f.blockchainClient, block: *blockC2.Hash(), at: func() { atMinedSet = f.unminedSince(t, f.bOnly) }} + bv.blockchainClient = probe + + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockC2.Hash())) + require.Equal(t, int32(1), probe.calls.Load(), "precondition: C' was set mined") + require.NotZero(t, atMinedSet, "B's tx is re-marked before C' gets mined_set") +} + +// TestSetTxMinedStatus_RepairsRecoveredSubtreeBeforeMinedSet: a subtree set aside whose +// reads have recovered is due again on the next block. That block's setTxMined must read it +// before setting mined_set, not leave it to a later slice that may lose to the pruner. +func TestSetTxMinedStatus_RepairsRecoveredSubtreeBeforeMinedSet(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + _ = f.fakeTipRepairClock(bv) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + bv.subtreeStore = f.shortReads(f.subtreeStore, 0, tipRepairSubtreeReadAttempts*tipRepairMaxAttempts) + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + require.NotNil(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending, "precondition: subtree 0 is set aside") + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: subtree 0's tx still looks mined") + + time.Sleep(10 * time.Millisecond) + + blockD2 := coinbaseOnlyBlock(t, blockC2.Hash(), f.nBits, 3, 0xd4) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD2, "test", blockchainoptions.WithSubtreesSet(true))) + + var atMinedSet uint32 + + probe := &minedSetProbe{ClientI: f.blockchainClient, block: *blockD2.Hash(), at: func() { atMinedSet = f.unminedSince(t, f.bOnly) }} + bv.blockchainClient = probe + + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockD2.Hash())) + require.Equal(t, int32(1), probe.calls.Load(), "precondition: D' was set mined") + require.NotZero(t, atMinedSet, "the recovered subtree's tx is re-marked before D' gets mined_set") +} + +// cappedHeadersClient rejects header and block-summary requests over the configured caps, +// as the blockchain gRPC server does; LocalClient enforces neither. +type cappedHeadersClient struct { + blockchain.ClientI + maxHeaders, maxBlocks uint64 +} + +func (c *cappedHeadersClient) GetBlockHeaders(ctx context.Context, hash *chainhash.Hash, n uint64) ([]*model.BlockHeader, []*model.BlockHeaderMeta, error) { + if n > c.maxHeaders { + return nil, nil, errors.NewInvalidArgumentError("[Blockchain][GetBlockHeaders] %d headers requested, maximum is %d", n, c.maxHeaders) + } + + return c.ClientI.GetBlockHeaders(ctx, hash, n) +} + +func (c *cappedHeadersClient) GetLastNBlocks(ctx context.Context, n int64, includeOrphans bool, fromHeight uint32) ([]*model.BlockInfo, error) { + if uint64(n) > c.maxBlocks { //nolint:gosec // test + return nil, errors.NewInvalidArgumentError("[Blockchain][GetLastNBlocks] invalid count %d, maximum is %d", n, c.maxBlocks) + } + + return c.ClientI.GetLastNBlocks(ctx, n, includeOrphans, fromHeight) +} + +// TestRepairOvertakenTips_ReadsWithinConfiguredCaps: blockchain_maxBlockHeadersPerRequest and +// blockchain_maxBlocksByHeightRange may be set below the repair window. The restart seed and +// the main-chain read must page within them, not fail every slice. +func TestRepairOvertakenTips_ReadsWithinConfiguredCaps(t *testing.T) { + f := newOvertakeFixture(t) + f.tSettings.BlockChain.MaxBlockHeadersPerRequest = 1 + f.tSettings.BlockChain.MaxBlocksByHeightRange = 1 + + bv := f.newBlockValidation(t) + client := &cappedHeadersClient{ClientI: f.blockchainClient, maxHeaders: 1, maxBlocks: 1} + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + // The seed runs in the first slice, then the main-chain read. + bv.tipRepairSeeded = false + + _, err := bv.tipRepairSlice(f.ctx, client) + require.NoError(t, err) + + _, tracked := bv.minedOnChain.snapshot()[*blockC2.Header.HashPrevBlock] + require.True(t, tracked, "B' is tracked from a main-chain read paged within the cap") + + for i := 0; i < 20; i++ { + if more, err := bv.tipRepairSlice(f.ctx, client); err != nil || !more { + require.NoError(t, err) + break + } + } + + require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired with every read within the caps") +} + +// TestRepairOvertakenTips_SetAsideSubtreesGauge: a subtree set aside cannot be repaired, and +// its txs' parents cannot be protected, until it reads again. Operators see how many there +// are, and the count drops once they are read. +func TestRepairOvertakenTips_SetAsideSubtreesGauge(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + clock := f.fakeTipRepairClock(bv) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + store := f.shortReads(f.subtreeStore, 0, 1<<30) + bv.subtreeStore = store + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + require.InDelta(t, 1, testutil.ToFloat64(prometheusBlockValidationTipRepairSubtreesSetAside), 0) + + store.short.Store(0) + clock.advance(tipRepairPendingMaxBackoff) + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.InDelta(t, 0, testutil.ToFloat64(prometheusBlockValidationTipRepairSubtreesSetAside), 0) +} diff --git a/stores/blockchain/mock_last_n_blocks_test.go b/stores/blockchain/mock_test.go similarity index 100% rename from stores/blockchain/mock_last_n_blocks_test.go rename to stores/blockchain/mock_test.go From fa5a02f1b98d3106cb3f9a1f2fa1711c024d0d67 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 15:18:16 +0100 Subject: [PATCH 26/33] fix(blockvalidation): page the restart seed past full heights, keep a repair error off the block being set mined The seed paged GetLastNBlocks with fromHeight, but the SQL blockchain store ignored fromHeight when orphans are included, so below a cap of 1000 every page returned the same top blocks and fork blocks further down were never seeded. The store now honours fromHeight there, as documented; the asset API's block list endpoints page with includeOrphans too. The seed also steps below a height once a page there holds nothing new, or is all at the height it started from, instead of stopping. The drain's error came from another tracked block, wrapped: a tracked block that is gone read as the block being set mined being gone, and the worker dropped its retry. It is now kept as text. The setting docs and CHANGELOG say the repair due runs in full before a block's mined status, not one slice at a time. --- CHANGELOG.md | 25 +++++-- .../services/blockvalidation_settings.md | 2 +- services/blockvalidation/BlockValidation.go | 13 ++-- .../blockvalidation/overtaken_tip_repair.go | 18 ++++- .../overtaken_tip_repair_test.go | 65 +++++++++++++++++++ settings.conf | 5 +- settings/blockvalidation_settings.go | 2 +- stores/blockchain/sql/GetLastNBlocks.go | 3 +- stores/blockchain/sql/GetLastNBlocks_test.go | 27 ++++++++ 9 files changed, 143 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 631a549bdd..935edc4737 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,12 +110,25 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea only the transactions it moved back and that are not mined on the new chain. Block validation now repairs a block it marked mined that a competing chain later overtakes, on its setMined worker, in slices of `blockvalidation_tipRepairSliceSize` transactions - (new setting, default 250000). Each slice can delay the next block's mined status by - one slice. The repair assumes a single block validation instance writes mined status - for the chain, and its state is in memory: fork blocks more than 10 below the tip at a - restart are not repaired. A deployment that runs more than one block validation - replica loses no data, but transactions can stay flagged unmined until block - assembly's next unmined load. No action needed for a single-instance deployment. (#2065) + (new setting, default 250000). A block's `mined_set` is written only once every repair + due on its chain has run, so the pruner cannot process that height first. After a + reorg, the new tip's `mined_set`, and with it the validation of its child, waits for + the overtaken blocks' repair, which costs about one setTxMined per overtaken block. A + subtree of an overtaken block that cannot be read is set aside, read again on every + block, and counted in `teranode_blockvalidation_tip_repair_subtrees_set_aside`; until it + reads, its transactions can look mined and their parents are not protected from the + pruner, so a gauge above zero for long needs looking at. The repair assumes a single + block validation instance writes mined status for the chain, and its state is in + memory: fork blocks more than 10 below the tip at a restart are not repaired. A + deployment that runs more than one block validation replica loses no data, but + transactions can stay flagged unmined until block assembly's next unmined load. No + action needed for a single-instance deployment. (#2065) +- **`GetLastNBlocks` honours `fromHeight` when orphans are included.** The SQL + blockchain store ignored `fromHeight` with `includeOrphans=true` and always returned + the highest blocks. It now returns only blocks at or below `fromHeight`, as documented. + This changes the asset API's block list endpoints when a client passes both + `includeOrphans` and `fromHeight`: they now page down instead of repeating the top of + the chain. (#2065) - **Catch-up quick validation waits for each block's `.subtree` writes before it commits the block, and a shutdown waits for a block's in-flight commit.** A block was stored with `subtrees_set` while its `.subtree` files could still be in flight, so a failed write diff --git a/docs/references/settings/services/blockvalidation_settings.md b/docs/references/settings/services/blockvalidation_settings.md index 135fc4978b..f41dd7ce29 100644 --- a/docs/references/settings/services/blockvalidation_settings.md +++ b/docs/references/settings/services/blockvalidation_settings.md @@ -69,7 +69,7 @@ loader (0, false and false), so for these keys the two agree. The | MaxBlocksBehindBlockAssembly | int | 20 | blockvalidation_maxBlocksBehindBlockAssembly | **CRITICAL** - Max blocks behind block assembly | | PeriodicProcessingInterval | time.Duration | 1m | blockvalidation_periodic_processing_interval | Periodic processing interval | | RecentBlockIDsLimit | uint64 | 50000 | blockvalidation_recentBlockIDsLimit | **CRITICAL** - Fast-path double-spend checking window | -| TipRepairSliceSize | int | 250000 | blockvalidation_tipRepairSliceSize | Transactions per overtaken-tip repair step on the setMined worker; bounds how long a step delays the next block's mined status | +| TipRepairSliceSize | int | 250000 | blockvalidation_tipRepairSliceSize | Transactions per overtaken-tip repair step on the setMined worker. Every repair due runs in full before the next block's mined status is written; between blocks, one slice is the most a step holds the worker | | MaxParallelForks | int | 4 | blockvalidation_max_parallel_forks | Maximum parallel fork processing | | MaxTrackedForks | int | 1000 | blockvalidation_max_tracked_forks | Maximum total forks tracked | | NearForkThreshold | int | 0 | blockvalidation_near_fork_threshold | Near fork detection (0=coinbase maturity/2) | diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 0ce38e0efe..539cb2b0c3 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1808,11 +1808,16 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain u.logger.Warnf("[setTxMined][%s] failed closing subtrees after setting mined: %v", block.Hash().String(), releaseErr) } - // The pruner processes a height only once its block has mined_set. Every repair this - // block's chain makes due runs first, so no parent of a tx that still looks mined off the - // chain is pruned before the tx is re-marked and its parents preserved. + // The pruner processes a height only once its block has mined_set. Every repair due on + // the chain as this block sees it runs first, so a tx that looks mined off the chain is + // re-marked, and its parents preserved by the pruner's next phase 1, before the pruner + // can process this height. An overtake that only becomes visible later is repaired by + // the worker's next slice or the next block's drain, within the slack between + // utxostore_blockHeightRetention and utxostore_unminedTxRetention. The repair's error + // is about another block, so it is kept as text: wrapped, a tracked block that is gone + // would read as this block being gone, and the worker would not retry it. if err = u.drainTipRepair(ctx, u.blockchainClient); err != nil { - return errors.NewProcessingError("[setTxMined][%s] overtaken-tip repair failed, mined_set not set", block.Hash().String(), err) + return errors.NewProcessingError("[setTxMined][%s] overtaken-tip repair failed, mined_set not set: %s", block.Hash().String(), err.Error()) } // update block mined_set to true diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index e735ff9cf9..a098cd2ff3 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -485,11 +485,25 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo u.minedOnChain.seed(*header.Hash(), meta.ID, meta.Height) } - if int64(len(recent)) < page || !added || recent[len(recent)-1].Height == 0 { + lowest := recent[len(recent)-1].Height + if int64(len(recent)) < page || lowest == 0 { break } - fromHeight = recent[len(recent)-1].Height + // The next page starts at the lowest height of this one, which it may share. A page + // with nothing new, or one all at the height it started from, would be read again + // as it is: step below it. Blocks at one height beyond the cap cannot be reached and + // are left out. + next := lowest + if !added || (fromHeight != 0 && lowest >= fromHeight) { + next = lowest - 1 + } + + if fromHeight != 0 && next >= fromHeight { + break // the store did not bound the page by fromHeight + } + + fromHeight = next } u.tipRepairSeeded = true diff --git a/services/blockvalidation/overtaken_tip_repair_test.go b/services/blockvalidation/overtaken_tip_repair_test.go index 07908fbc74..bed1d0cd41 100644 --- a/services/blockvalidation/overtaken_tip_repair_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -2225,3 +2225,68 @@ func TestRepairOvertakenTips_SetAsideSubtreesGauge(t *testing.T) { require.NoError(t, err) require.InDelta(t, 0, testutil.ToFloat64(prometheusBlockValidationTipRepairSubtreesSetAside), 0) } + +// TestRepairOvertakenTips_SeedPagesForkBlocksWithinCap: the restart seed finds fork blocks +// near the tip with GetLastNBlocks, which blockchain_maxBlocksByHeightRange may cap below the +// seed's reach. It must page down past full pages of blocks at one height, not stop there: +// with a cap of 2, the fork blocks X at height 2 and Y at height 1 each sit below a full page. +func TestRepairOvertakenTips_SeedPagesForkBlocksWithinCap(t *testing.T) { + f := newOvertakeFixture(t) + f.tSettings.BlockChain.MaxBlocksByHeightRange = 2 + + bv := f.newBlockValidation(t) + client := &cappedHeadersClient{ClientI: f.blockchainClient, maxHeaders: tipRepairWindow, maxBlocks: 2} + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + time.Sleep(10 * time.Millisecond) + + // genesis - B - D - E, with forks D - F, B - X and genesis - Y. + blockY := coinbaseOnlyBlock(t, f.tSettings.ChainCfgParams.GenesisHash, f.nBits, 1, 0xb0) + blockD := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xd0) + blockX := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xa0) + blockE := coinbaseOnlyBlock(t, blockD.Hash(), f.nBits, 3, 0xe0) + blockF := coinbaseOnlyBlock(t, blockD.Hash(), f.nBits, 3, 0xf0) + + for _, block := range []*model.Block{blockY, blockD, blockX, blockE, blockF} { + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, block, "test", blockchainoptions.WithSubtreesSet(true))) + } + + bv.tipRepairSeeded = false + + _, err := bv.tipRepairSlice(f.ctx, client) + require.NoError(t, err) + + _, tracked := bv.minedOnChain.snapshot()[*blockX.Hash()] + require.True(t, tracked, "fork block X, below a full page at height 3, is seeded") + + _, tracked = bv.minedOnChain.snapshot()[*blockY.Hash()] + require.True(t, tracked, "fork block Y, below a full page at height 2, is seeded") +} + +// TestSetTxMinedStatus_FailedRepairLeavesMinedSetUnset: when the repair keeps failing, the +// block is not set mined, so the pruner cannot process its height. The error is the +// repair's, not the block's: a tracked block that is gone must not make the worker take the +// block it is setting mined for gone too and drop its retry. +func TestSetTxMinedStatus_FailedRepairLeavesMinedSetUnset(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + // A tracked block the blockchain no longer has: every attempt to repair it fails. + bv.minedOnChain.track(chainhash.HashH([]byte("gone")), 999, 1) + + time.Sleep(10 * time.Millisecond) + + blockD := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xd0) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD, "test", blockchainoptions.WithSubtreesSet(true))) + + err := bv.setTxMinedStatus(f.ctx, blockD.Hash()) + require.Error(t, err) + require.False(t, errors.Is(err, errors.ErrBlockNotFound), "the error is not the block's own: %v", err) + + _, meta, err := f.blockchainClient.GetBlockHeader(f.ctx, blockD.Hash()) + require.NoError(t, err) + require.False(t, meta.MinedSet, "mined_set stays unset while the repair fails") +} diff --git a/settings.conf b/settings.conf index 8dc378d1a6..afc632ec44 100644 --- a/settings.conf +++ b/settings.conf @@ -645,8 +645,9 @@ blockvalidation_recentBlockIDsLimit.docker.m = 10000 blockvalidation_recentBlockIDsLimit.dev = 1000 # Transactions re-marked per step of the overtaken-tip repair, which runs on the setMined -# worker between blocks (issue 2051). Bounds how long one step delays the next block's -# mined status. +# worker (issue 2051). Every repair due runs in full before a block's mined status is +# written, so the pruner cannot process that height first; between blocks, one slice is +# the most a step holds the worker. blockvalidation_tipRepairSliceSize = 250000 # Skip the per-transaction UTXO lock/unlock during quick-validation catchup (default: false). diff --git a/settings/blockvalidation_settings.go b/settings/blockvalidation_settings.go index c79b0c5926..f2f3d512ad 100644 --- a/settings/blockvalidation_settings.go +++ b/settings/blockvalidation_settings.go @@ -73,7 +73,7 @@ type BlockValidationSettings struct { MaxBlocksBehindBlockAssembly int `key:"blockvalidation_maxBlocksBehindBlockAssembly" desc:"Maximum blocks behind Block Assembly" default:"20" category:"BlockValidation" usage:"Sync tolerance" type:"int" longdesc:"### Purpose\nSets the maximum blocks Block Validation can lag behind Block Assembly.\n\n### How It Works\n- Block Assembly may produce blocks faster than Block Validation can process\n- When this threshold is exceeded, catchup mode or waiting is triggered\n- Prevents unbounded queue growth between services\n\n### Recommendations\n- **20** (default) - Reasonable buffer for processing delays\n- Increase for systems with variable validation times\n- Decrease to maintain tighter synchronization"` PeriodicProcessingInterval time.Duration `key:"blockvalidation_periodic_processing_interval" desc:"Interval for periodic processing" default:"1m" category:"BlockValidation" usage:"Background task frequency" type:"duration" longdesc:"### Purpose\nSets the interval for periodic background processing tasks.\n\n### Tasks Performed\n- Checking mined status of pending blocks\n- Cache cleanup and maintenance\n- Resource cleanup and garbage collection\n\n### Recommendations\n- **1m** (default) - Regular maintenance without excessive overhead\n- Decrease for faster status updates (more CPU usage)\n- Increase for reduced background activity"` RecentBlockIDsLimit uint64 `key:"blockvalidation_recentBlockIDsLimit" desc:"Maximum recent block IDs to load" default:"50000" category:"BlockValidation" usage:"Memory limit for recent block IDs" type:"uint64" longdesc:"### Purpose\nLimits the number of recent block IDs loaded into memory for double-spend checking.\n\n### How It Works\nRecent block IDs are kept in memory to support fast double-spend detection. This setting prevents unbounded memory growth.\n\n### Recommendations\n- **50000** (default) - Covers typical scenarios\n- Increase for deeper history requirements\n- Decrease for memory-constrained environments"` - TipRepairSliceSize int `key:"blockvalidation_tipRepairSliceSize" desc:"Transactions re-marked per overtaken-tip repair slice" default:"250000" category:"BlockValidation" usage:"Bounds how long one repair slice holds the setMined worker" type:"int" longdesc:"### Purpose\nBounds the work of one slice of the overtaken-tip repair.\n\n### How It Works\nWhen a block whose transactions were marked mined is overtaken before block assembly moved onto it, block validation re-marks the transactions that are not mined on the new chain. It does this on the setMined worker, between blocks, a slice of transactions at a time, so the next block's mined status is delayed by at most one slice.\n\n### Recommendations\n- **250000** (default)\n- Decrease if setTxMined latency matters more than repair speed"` + TipRepairSliceSize int `key:"blockvalidation_tipRepairSliceSize" desc:"Transactions re-marked per overtaken-tip repair slice" default:"250000" category:"BlockValidation" usage:"Bounds the batch each repair step reads and writes" type:"int" longdesc:"### Purpose\nBounds the work of one slice of the overtaken-tip repair.\n\n### How It Works\nWhen a block whose transactions were marked mined is overtaken before block assembly moved onto it, block validation re-marks the transactions that are not mined on the new chain. It does this on the setMined worker a slice of transactions at a time. Every repair due runs in full before the next block's mined status is written, so the pruner cannot process that height first: after a reorg, the next block's mined status waits for the whole repair, whatever the slice size. Between blocks, one slice is the most a repair step holds the worker.\n\n### Recommendations\n- **250000** (default)\n- Decrease to lower the memory one step uses"` // Catchup configuration CatchupChBufferSize int `key:"blockvalidation_catchupCh_buffer_size" desc:"Buffer size for catchup channel" default:"100" category:"BlockValidation" usage:"Queue for catchup blocks" type:"int" longdesc:"### Purpose\nSets the buffer size for the channel receiving catchup block notifications.\n\n### How It Works\nDuring catchup mode, blocks are queued for processing. This buffer prevents blocking when multiple blocks arrive quickly.\n\n### Recommendations\n- **100** (default) - Handles typical catchup scenarios\n- Increase if experiencing backpressure during large syncs"` UseCatchupWhenBehind bool `key:"blockvalidation_useCatchupWhenBehind" desc:"Enable automatic catchup mode" default:"false" category:"BlockValidation" usage:"Auto-switch to catchup" type:"bool" longdesc:"### Purpose\nAutomatically switches to optimized catchup mode when the node falls behind the network.\n\n### How It Works\n- Monitors chain height relative to known network tip\n- When behind by threshold, switches to catchup mode\n- Catchup uses batch fetching and quick validation for faster sync\n- Returns to normal mode once caught up\n\n### Values\n- **false** (default) - Always use normal validation\n- **true** - Enable automatic catchup mode switching\n\n### Recommendations\n- Enable for nodes that may experience periods of being offline\n- Keep disabled for always-connected mining nodes"` diff --git a/stores/blockchain/sql/GetLastNBlocks.go b/stores/blockchain/sql/GetLastNBlocks.go index 8a7149d5bf..0a1985b7ca 100644 --- a/stores/blockchain/sql/GetLastNBlocks.go +++ b/stores/blockchain/sql/GetLastNBlocks.go @@ -88,10 +88,11 @@ func (s *SQL) GetLastNBlocks(ctx context.Context, n int64, includeOrphans bool, ,b.inserted_at FROM blocks b WHERE invalid = false + AND ($2 = 0 OR height <= $2) ORDER BY height DESC LIMIT $1 ` - args = []any{n} + args = []any{n, fromHeight} } else if fromHeight > 0 { if s.mainChainRebuilding.Load() > 0 { // When paginating, cap CTE walk depth to the lower of tip height and fromHeight, diff --git a/stores/blockchain/sql/GetLastNBlocks_test.go b/stores/blockchain/sql/GetLastNBlocks_test.go index d22244a91d..c67f5e4dd2 100644 --- a/stores/blockchain/sql/GetLastNBlocks_test.go +++ b/stores/blockchain/sql/GetLastNBlocks_test.go @@ -157,6 +157,33 @@ func TestSQLGetLastNBlocks(t *testing.T) { assert.GreaterOrEqual(t, len(blocks), 4) // Genesis + 3 blocks + alternative }) + t.Run("get last blocks with includeOrphans=true and fromHeight", func(t *testing.T) { + storeURL, err := url.Parse("sqlitememory:///") + require.NoError(t, err) + + s, err := New(ulogger.TestLogger{}, storeURL, tSettings) + require.NoError(t, err) + + _, _, err = s.StoreBlock(context.Background(), block1, "") + require.NoError(t, err) + _, _, err = s.StoreBlock(context.Background(), block2, "") + require.NoError(t, err) + _, _, err = s.StoreBlock(context.Background(), block3, "") + require.NoError(t, err) + _, _, err = s.StoreBlock(context.Background(), blockAlternative2, "") + require.NoError(t, err) + + // fromHeight bounds the orphan-inclusive read as it does the main-chain one, so a + // caller can page down from the tip: block3 is above it, both blocks at 2 are not. + blocks, err := s.GetLastNBlocks(context.Background(), 2, true, 2) + require.NoError(t, err) + require.Len(t, blocks, 2) + + for _, b := range blocks { + require.Equal(t, uint32(2), b.Height) + } + }) + t.Run("cache hit", func(t *testing.T) { storeURL, err := url.Parse("sqlitememory:///") require.NoError(t, err) From 1b79483f27100b21f3405893c7403411ac674f2f Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 15:30:00 +0100 Subject: [PATCH 27/33] test(settings): count the blockvalidation settings row main and this PR each added --- settings/settings_doc_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings/settings_doc_test.go b/settings/settings_doc_test.go index 6c8e23a37b..15dbbaeb43 100644 --- a/settings/settings_doc_test.go +++ b/settings/settings_doc_test.go @@ -901,7 +901,7 @@ var checkedRowsPerDoc = map[string]int{ "blockassembly_settings.md": 44, "blockchain_settings.md": 11, "blockpersister_settings.md": 9, - "blockvalidation_settings.md": 71, + "blockvalidation_settings.md": 72, "coinbase_settings.md": 21, "faucet_settings.md": 1, "legacy_settings.md": 28, From 835b93b12f88f6a42e7172d496886834c6ccf1ea Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 16:30:39 +0100 Subject: [PATCH 28/33] fix(pruner): hold all deletion while an overtaken-tip repair is unfinished, so mined_set no longer waits on repair The drain before SetBlockMinedSet made every new block wait on the repair of old overtaken blocks, so a repair that kept failing stopped the chain, and it still let a subtree that could not be read reach mined_set with its parents unprotected. The pruner now has its own gate instead. Block validation records the overtaken blocks it has not finished repairing (off the main chain and not done, pending or with subtrees set aside) under the blockchain state key OvertakenTipRepairHold. It brings the record up to date before every SetBlockMinedSet, which only reads the chain and writes the record, and after every repair slice. The pruner reads it after the mined_set wait: while it lists any block, it skips blob deletion and phase 2, does not advance its height, and resumes on its own once the record is empty; a record it cannot read counts as held. Phase 1 still runs. The record survives a restart: the seed reads it back before anything can write it, so held blocks are repaired even when they are deeper than the seed reaches. The production drain is gone; mined_set reflects only the block's own write. ParentOutputsForValidation's lock retry is now tested through the store with a read hook, as the blockchain store's reconcileHook does. --- CHANGELOG.md | 31 ++-- .../services/blockvalidation_settings.md | 2 +- model/overtaken_tip_repair_hold.go | 58 +++++++ model/overtaken_tip_repair_hold_test.go | 35 ++++ services/blockvalidation/BlockValidation.go | 22 +-- .../blockvalidation/overtaken_tip_repair.go | 161 +++++++++++++++--- .../overtaken_tip_repair_test.go | 142 +++++++++++---- services/pruner/metrics.go | 12 ++ services/pruner/worker.go | 72 +++++++- services/pruner/worker_test.go | 141 +++++++++++++++ settings.conf | 5 +- settings/blockvalidation_settings.go | 2 +- stores/utxo/sql/parent_outputs.go | 6 + stores/utxo/sql/parent_outputs_test.go | 28 +++ stores/utxo/sql/sql.go | 23 +-- 15 files changed, 635 insertions(+), 105 deletions(-) create mode 100644 model/overtaken_tip_repair_hold.go create mode 100644 model/overtaken_tip_repair_hold_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 728513c5ae..aaa78ec49f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,19 +110,24 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea only the transactions it moved back and that are not mined on the new chain. Block validation now repairs a block it marked mined that a competing chain later overtakes, on its setMined worker, in slices of `blockvalidation_tipRepairSliceSize` transactions - (new setting, default 250000). A block's `mined_set` is written only once every repair - due on its chain has run, so the pruner cannot process that height first. After a - reorg, the new tip's `mined_set`, and with it the validation of its child, waits for - the overtaken blocks' repair, which costs about one setTxMined per overtaken block. A - subtree of an overtaken block that cannot be read is set aside, read again on every - block, and counted in `teranode_blockvalidation_tip_repair_subtrees_set_aside`; until it - reads, its transactions can look mined and their parents are not protected from the - pruner, so a gauge above zero for long needs looking at. The repair assumes a single - block validation instance writes mined status for the chain, and its state is in - memory: fork blocks more than 10 below the tip at a restart are not repaired. A - deployment that runs more than one block validation replica loses no data, but - transactions can stay flagged unmined until block assembly's next unmined load. No - action needed for a single-instance deployment. (#2065) + (new setting, default 250000); each slice can delay the next block's mined status by + one slice. **The pruner deletes nothing while a repair is unfinished.** Block validation + records the overtaken blocks it has not finished repairing under the blockchain state + key `OvertakenTipRepairHold`, before it sets the mined status of the block that + overtook them, and the pruner skips UTXO deletion (phase 2) and blob deletion while any + is listed; parent preservation (phase 1) still runs, and deletion resumes on its own + once the repair finishes. A subtree of an overtaken block that cannot be read is read + again on every block and holds pruning until it reads or the block leaves the + 1000-block repair window. `teranode_pruner_overtaken_tip_repair_held_blocks` and + `teranode_blockvalidation_tip_repair_subtrees_set_aside` show a hold; a hold that stays + above zero means a subtree file is corrupt or a store keeps failing, and disk use grows + until it is resolved. A repair that keeps failing no longer stops healthy blocks: their + mined status does not wait for it. The hold survives a restart, and the blocks it lists + are repaired after it. **A single block validation instance must write mined status + for the chain**: the hold is one record, so two instances can release each other's + hold. A block overtaken while block validation is down and more than 10 blocks below + the tip when it starts is not repaired. No action needed for a single-instance + deployment. (#2065) - **`GetLastNBlocks` honours `fromHeight` when orphans are included.** The SQL blockchain store ignored `fromHeight` with `includeOrphans=true` and always returned the highest blocks. It now returns only blocks at or below `fromHeight`, as documented. diff --git a/docs/references/settings/services/blockvalidation_settings.md b/docs/references/settings/services/blockvalidation_settings.md index 52b18e4600..a8f425dc73 100644 --- a/docs/references/settings/services/blockvalidation_settings.md +++ b/docs/references/settings/services/blockvalidation_settings.md @@ -69,7 +69,7 @@ loader (0, false and false), so for these keys the two agree. The | MaxBlocksBehindBlockAssembly | int | 20 | blockvalidation_maxBlocksBehindBlockAssembly | **CRITICAL** - Max blocks behind block assembly | | PeriodicProcessingInterval | time.Duration | 1m | blockvalidation_periodic_processing_interval | Periodic processing interval | | RecentBlockIDsLimit | uint64 | 50000 | blockvalidation_recentBlockIDsLimit | **CRITICAL** - Fast-path double-spend checking window | -| TipRepairSliceSize | int | 250000 | blockvalidation_tipRepairSliceSize | Transactions per overtaken-tip repair step on the setMined worker. Every repair due runs in full before the next block's mined status is written; between blocks, one slice is the most a step holds the worker | +| TipRepairSliceSize | int | 250000 | blockvalidation_tipRepairSliceSize | Transactions per overtaken-tip repair step on the setMined worker; bounds how long a step delays the next block's mined status. The pruner deletes nothing until the repair has finished | | MaxParallelForks | int | 4 | blockvalidation_max_parallel_forks | Maximum parallel fork processing | | MaxTrackedForks | int | 1000 | blockvalidation_max_tracked_forks | Maximum total forks tracked | | NearForkThreshold | int | 0 | blockvalidation_near_fork_threshold | Near fork detection (0=coinbase maturity/2) | diff --git a/model/overtaken_tip_repair_hold.go b/model/overtaken_tip_repair_hold.go new file mode 100644 index 0000000000..deaab39446 --- /dev/null +++ b/model/overtaken_tip_repair_hold.go @@ -0,0 +1,58 @@ +package model + +import ( + "encoding/binary" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/bsv-blockchain/teranode/errors" +) + +// OvertakenTipRepairHoldKey is the blockchain state key under which block validation records +// the overtaken blocks whose repair has not finished. While the record lists any, the pruner +// deletes nothing: their transactions may look mined while off the longest chain, so the +// pruner cannot tell which parents they still need. An absent or empty record holds nothing. +const OvertakenTipRepairHoldKey = "OvertakenTipRepairHold" + +// heldBlockSize is the encoded size of one HeldBlock: hash, id, height. +const heldBlockSize = chainhash.HashSize + 4 + 4 + +// HeldBlock is an overtaken block whose repair has not finished. +type HeldBlock struct { + Hash chainhash.Hash + ID uint32 + Height uint32 +} + +// EncodeOvertakenTipRepairHold encodes held blocks for OvertakenTipRepairHoldKey. No blocks +// encode as an empty record. +func EncodeOvertakenTipRepairHold(held []HeldBlock) []byte { + data := make([]byte, 0, len(held)*heldBlockSize) + + for _, b := range held { + data = append(data, b.Hash[:]...) + data = binary.LittleEndian.AppendUint32(data, b.ID) + data = binary.LittleEndian.AppendUint32(data, b.Height) + } + + return data +} + +// DecodeOvertakenTipRepairHold decodes a record written by EncodeOvertakenTipRepairHold. +func DecodeOvertakenTipRepairHold(data []byte) ([]HeldBlock, error) { + if len(data)%heldBlockSize != 0 { + return nil, errors.NewProcessingError("invalid overtaken-tip repair hold: %d bytes is not a whole number of %d-byte entries", len(data), heldBlockSize) + } + + held := make([]HeldBlock, 0, len(data)/heldBlockSize) + + for off := 0; off < len(data); off += heldBlockSize { + var b HeldBlock + + copy(b.Hash[:], data[off:off+chainhash.HashSize]) + b.ID = binary.LittleEndian.Uint32(data[off+chainhash.HashSize:]) + b.Height = binary.LittleEndian.Uint32(data[off+chainhash.HashSize+4:]) + held = append(held, b) + } + + return held, nil +} diff --git a/model/overtaken_tip_repair_hold_test.go b/model/overtaken_tip_repair_hold_test.go new file mode 100644 index 0000000000..0ce7979e58 --- /dev/null +++ b/model/overtaken_tip_repair_hold_test.go @@ -0,0 +1,35 @@ +package model + +import ( + "testing" + + "github.com/bsv-blockchain/go-bt/v2/chainhash" + "github.com/stretchr/testify/require" +) + +func TestOvertakenTipRepairHoldRoundTrip(t *testing.T) { + held := []HeldBlock{ + {Hash: chainhash.HashH([]byte("a")), ID: 7, Height: 100}, + {Hash: chainhash.HashH([]byte("b")), ID: 9, Height: 98}, + } + + decoded, err := DecodeOvertakenTipRepairHold(EncodeOvertakenTipRepairHold(held)) + require.NoError(t, err) + require.Equal(t, held, decoded) +} + +func TestOvertakenTipRepairHoldEmpty(t *testing.T) { + encoded := EncodeOvertakenTipRepairHold(nil) + require.Empty(t, encoded, "no hold is an empty record") + + decoded, err := DecodeOvertakenTipRepairHold(encoded) + require.NoError(t, err) + require.Empty(t, decoded) +} + +func TestOvertakenTipRepairHoldRejectsTruncated(t *testing.T) { + encoded := EncodeOvertakenTipRepairHold([]HeldBlock{{Hash: chainhash.HashH([]byte("a")), ID: 7, Height: 100}}) + + _, err := DecodeOvertakenTipRepairHold(encoded[:len(encoded)-1]) + require.Error(t, err) +} diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index f0cedd1195..def3074cb3 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -343,6 +343,10 @@ type BlockValidation struct { // tipRepairMu), so a best block that only extends the chain costs one call. tipRepairMainChain map[uint32]bool tipRepairBestHash chainhash.Hash + // tipRepairHold is the pruning hold last written or read back, once tipRepairHoldWritten + // is set (guarded by tipRepairMu). + tipRepairHold []byte + tipRepairHoldWritten bool // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. tipRepairRetry atomic.Bool // tipRepairWake requests a slice when the earliest pass over subtrees set aside comes @@ -1814,16 +1818,14 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain u.logger.Warnf("[setTxMined][%s] failed closing subtrees after setting mined: %v", block.Hash().String(), releaseErr) } - // The pruner processes a height only once its block has mined_set. Every repair due on - // the chain as this block sees it runs first, so a tx that looks mined off the chain is - // re-marked, and its parents preserved by the pruner's next phase 1, before the pruner - // can process this height. An overtake that only becomes visible later is repaired by - // the worker's next slice or the next block's drain, within the slack between - // utxostore_blockHeightRetention and utxostore_unminedTxRetention. The repair's error - // is about another block, so it is kept as text: wrapped, a tracked block that is gone - // would read as this block being gone, and the worker would not retry it. - if err = u.drainTipRepair(ctx, u.blockchainClient); err != nil { - return errors.NewProcessingError("[setTxMined][%s] overtaken-tip repair failed, mined_set not set: %s", block.Hash().String(), err.Error()) + // The pruner processes a height only once its block has mined_set. A block the chain + // has overtaken goes into the pruning hold first, so the pruner deletes nothing until + // its repair has finished, however long that takes. This reads the chain and writes the + // hold; it repairs nothing, so a repair that keeps failing does not stop this block. Its + // error is kept as text: it may be about another block, and wrapped, a block that is + // gone would read as this block being gone, and the worker would not retry it. + if err = u.refreshRepairHold(ctx, u.blockchainClient); err != nil { + return errors.NewServiceError("[setTxMined][%s] failed to update the overtaken-tip pruning hold, mined_set not set: %s", block.Hash().String(), err.Error()) } // update block mined_set to true diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index a098cd2ff3..4438af28c1 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -2,7 +2,12 @@ package blockvalidation import ( "bufio" + "bytes" + "cmp" "context" + "database/sql" + "slices" + "strings" "sync" "time" @@ -336,31 +341,53 @@ type tipRepairJob struct { height uint32 } -// drainTipRepair runs repair slices until none is due. setTxMinedStatus calls it before -// setting a block's mined_set, which is what lets the pruner process that height: a repair -// the chain has made due is finished first, so its txs are re-marked, and their parents -// preserved by the pruner's next phase 1, before any parent can be pruned. A pass over -// subtrees set aside is due on every new block, so a subtree whose reads have recovered is -// read here too. One that still cannot be read stays set aside: its txs cannot be named, and -// tipRepairSubtreesSetAside reports it. A slice that keeps failing returns its error after -// tipRepairMaxAttempts in a row; the caller leaves mined_set unset, to be retried. -func (u *BlockValidation) drainTipRepair(ctx context.Context, blockchainClient blockchain.ClientI) error { - for failures := 0; ; { - more, err := u.tipRepairSlice(ctx, blockchainClient) - if err != nil { - if failures++; failures >= tipRepairMaxAttempts || ctx.Err() != nil { - return err - } +// writeRepairHoldLocked records in the blockchain state the tracked blocks that are off the +// main chain and not yet repaired, pending or with subtrees set aside: their transactions may +// look mined while off the chain, and the pruner deletes nothing while any is listed. A block +// on the chain owing a remine is not listed: its transactions look unmined while mined, which +// only keeps their parents longer. The record is written only when it changes. It is not +// written before the tracker is seeded, so a hold recorded before a restart is read back +// first (seedMinedOnChainTrackerLocked). +func (u *BlockValidation) writeRepairHoldLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { + if !u.tipRepairSeeded { + return nil + } - continue - } + held := make([]model.HeldBlock, 0) - failures = 0 + for hash, b := range u.minedOnChain.snapshot() { + if !u.tipRepairMainChain[b.id] && !b.done { + held = append(held, model.HeldBlock{Hash: hash, ID: b.id, Height: b.height}) + } + } - if !more { - return nil + slices.SortFunc(held, func(a, b model.HeldBlock) int { + if c := cmp.Compare(a.Height, b.Height); c != 0 { + return c } + + return bytes.Compare(a.Hash[:], b.Hash[:]) + }) + + data := model.EncodeOvertakenTipRepairHold(held) + if u.tipRepairHoldWritten && bytes.Equal(data, u.tipRepairHold) { + return nil + } + + if err := blockchainClient.SetState(ctx, model.OvertakenTipRepairHoldKey, data); err != nil { + return errors.NewServiceError("[repairOvertakenTips] failed to record the pruning hold", err) + } + + if len(held) > 0 { + u.logger.Warnf("[repairOvertakenTips] holding pruning for %d overtaken blocks still being repaired, lowest at height %d", len(held), held[0].Height) + } else if u.tipRepairHoldWritten && len(u.tipRepairHold) > 0 { + u.logger.Infof("[repairOvertakenTips] every overtaken block is repaired, pruning released") } + + u.tipRepairHold = data + u.tipRepairHoldWritten = true + + return nil } // signalTipRepair asks the setMined worker for a repair slice. Never blocks; a pending @@ -416,6 +443,13 @@ func (u *BlockValidation) runTipRepairSlice(ctx context.Context, blockchainClien // main-chain blocks inside the window, and the fork blocks within tipRepairSeedForkDepth of // the best height, which may have been the tip before the restart. func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { + // The blocks held before the restart first: their repair had not finished, and the + // window and fork depth below may no longer reach them. Read before anything here can + // write the hold again. + if err := u.seedHeldBlocksLocked(ctx, blockchainClient); err != nil { + return err + } + bestHeader, bestMeta, err := blockchainClient.GetBestBlockHeader(ctx) if err != nil { return errors.NewServiceError("[repairOvertakenTips] seed: failed to get best block header", err) @@ -511,6 +545,35 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo return nil } +// seedHeldBlocksLocked tracks the blocks a hold recorded before the restart lists. No record +// means a node that has never held pruning. +func (u *BlockValidation) seedHeldBlocksLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { + data, err := blockchainClient.GetState(ctx, model.OvertakenTipRepairHoldKey) + if err != nil { + if errors.Is(err, errors.ErrNotFound) || strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + return nil + } + + return errors.NewServiceError("[repairOvertakenTips] seed: failed to read the pruning hold", err) + } + + held, err := model.DecodeOvertakenTipRepairHold(data) + if err != nil { + // Unreadable, it protects nothing; the hold is written again from the tracker. + u.logger.Errorf("[repairOvertakenTips] seed: ignoring an unreadable pruning hold: %v", err) + return nil + } + + for _, b := range held { + u.minedOnChain.seed(b.Hash, b.ID, b.Height) + } + + u.tipRepairHold = data + u.tipRepairHoldWritten = true + + return nil +} + // mainChainHeaders returns the tipRepairWindow headers ending at best, best first, read in // pages within blockchain_maxBlockHeadersPerRequest, which the blockchain service enforces // and may be set below the window. @@ -582,26 +645,66 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b prometheusBlockValidationTipRepairSubtreesSetAside.Set(float64(u.minedOnChain.setAsideSubtrees())) }() + tracked, onMainChain, bestHeight, active, err := u.tipRepairHousekeepingLocked(ctx, blockchainClient) + if err != nil || !active { + return false, err + } + + // The step may finish a block's repair, or set its subtrees aside: record the hold as it + // stands after it. + defer func() { + if holdErr := u.writeRepairHoldLocked(ctx, blockchainClient); holdErr != nil && err == nil { + more, err = true, holdErr + } + }() + + return u.tipRepairStepLocked(ctx, blockchainClient, tracked, onMainChain, bestHeight) +} + +// refreshRepairHold brings the pruning hold up to date with the chain as it is now, without +// repairing anything: setTxMinedStatus calls it before setting a block's mined_set, which is +// what lets the pruner process that height. A block the chain has just overtaken is held +// before the pruner can see the new height, so the pruner deletes nothing until its repair +// has finished. It reads the chain from the blockchain service and writes the hold there, the +// same service SetBlockMinedSet writes to; it does no repair work, so a repair that fails does +// not stop healthy blocks from being set mined. +func (u *BlockValidation) refreshRepairHold(ctx context.Context, blockchainClient blockchain.ClientI) error { + u.tipRepairMu.Lock() + defer u.tipRepairMu.Unlock() + + if _, _, _, active, err := u.tipRepairHousekeepingLocked(ctx, blockchainClient); err != nil || !active { + return err + } + + return u.writeRepairHoldLocked(ctx, blockchainClient) +} + +// tipRepairHousekeepingLocked seeds the tracker after a restart, reads the main chain, and +// drops or reopens tracked blocks as the chain has moved. active is false until this process +// has tracked a block, when there is nothing to do. +func (u *BlockValidation) tipRepairHousekeepingLocked(ctx context.Context, blockchainClient blockchain.ClientI) (tracked map[chainhash.Hash]minedOnChainBlock, + onMainChain map[uint32]bool, bestHeight uint32, active bool, err error) { // Nothing to do until this process has written a block as on the longest chain. The // restart seed waits for that too: after a restart the next block's setTxMined comes // within one block interval, and a process that never marks blocks mined (or a test - // harness that never runs setTxMined) makes no blockchain calls here at all. + // harness that never runs setTxMined) makes no blockchain calls here at all. A hold + // recorded before the restart stays as it is until then. if !u.tipRepairSeeded && u.minedOnChain.empty() { - return false, nil + return nil, nil, 0, false, nil } if !u.tipRepairSeeded { if err = u.seedMinedOnChainTrackerLocked(ctx, blockchainClient); err != nil { - return false, err + return nil, nil, 0, false, err } } - onMainChain, bestHeight, err := u.mainChainIDsLocked(ctx, blockchainClient) + onMainChain, bestHeight, err = u.mainChainIDsLocked(ctx, blockchainClient) if err != nil { - return false, err + return nil, nil, 0, false, err } - tracked := u.minedOnChain.snapshot() + tracked = u.minedOnChain.snapshot() for hash, b := range tracked { switch { @@ -640,6 +743,12 @@ func (u *BlockValidation) tipRepairSlice(ctx context.Context, blockchainClient b } } + return tracked, onMainChain, bestHeight, true, nil +} + +// tipRepairStepLocked does one step of the job in progress, or starts the next job. +func (u *BlockValidation) tipRepairStepLocked(ctx context.Context, blockchainClient blockchain.ClientI, tracked map[chainhash.Hash]minedOnChainBlock, + onMainChain map[uint32]bool, bestHeight uint32) (more bool, err error) { if job := u.tipRepairJob; job != nil { b, ok := tracked[job.hash] diff --git a/services/blockvalidation/overtaken_tip_repair_test.go b/services/blockvalidation/overtaken_tip_repair_test.go index bed1d0cd41..c683167b2d 100644 --- a/services/blockvalidation/overtaken_tip_repair_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -2092,57 +2092,88 @@ func (c *minedSetProbe) SetBlockMinedSet(ctx context.Context, blockHash *chainha return c.ClientI.SetBlockMinedSet(ctx, blockHash) } -// TestSetTxMinedStatus_RepairsOvertakenBlockBeforeMinedSet: the pruner processes a height -// only once its block has mined_set. So the setTxMined of the block that overtook B must -// finish B's repair before it sets mined_set, or the pruner can delete a parent of a tx of -// B that still looks mined. -func TestSetTxMinedStatus_RepairsOvertakenBlockBeforeMinedSet(t *testing.T) { +// heldBlocks returns the blocks the pruning hold in the blockchain state lists. +func (f *overtakeFixture) heldBlocks(t *testing.T) []chainhash.Hash { + t.Helper() + + data, err := f.blockchainClient.GetState(f.ctx, model.OvertakenTipRepairHoldKey) + if err != nil { + return nil + } + + held, err := model.DecodeOvertakenTipRepairHold(data) + require.NoError(t, err) + + hashes := make([]chainhash.Hash, 0, len(held)) + for _, b := range held { + hashes = append(hashes, b.Hash) + } + + return hashes +} + +// TestSetTxMinedStatus_HoldsOvertakenBlockBeforeMinedSet: the pruner processes a height only +// once its block has mined_set. So the setTxMined of the block that overtook B must put B in +// the pruning hold before it sets mined_set, or the pruner can delete a parent of a tx of B +// that still looks mined. It must not wait for B's repair to do so. +func TestSetTxMinedStatus_HoldsOvertakenBlockBeforeMinedSet(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) blockC2 := f.overtakeB(t) - var atMinedSet uint32 + var heldAtMinedSet []chainhash.Hash - probe := &minedSetProbe{ClientI: f.blockchainClient, block: *blockC2.Hash(), at: func() { atMinedSet = f.unminedSince(t, f.bOnly) }} + probe := &minedSetProbe{ClientI: f.blockchainClient, block: *blockC2.Hash(), at: func() { heldAtMinedSet = f.heldBlocks(t) }} bv.blockchainClient = probe require.NoError(t, bv.setTxMinedStatus(f.ctx, blockC2.Hash())) require.Equal(t, int32(1), probe.calls.Load(), "precondition: C' was set mined") - require.NotZero(t, atMinedSet, "B's tx is re-marked before C' gets mined_set") + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, heldAtMinedSet, "B is held before C' gets mined_set") + require.Zero(t, f.unminedSince(t, f.bOnly), "the hold does not wait for B's repair") + + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "precondition: B is repaired") + require.Empty(t, f.heldBlocks(t), "the hold is released once B is repaired") } -// TestSetTxMinedStatus_RepairsRecoveredSubtreeBeforeMinedSet: a subtree set aside whose -// reads have recovered is due again on the next block. That block's setTxMined must read it -// before setting mined_set, not leave it to a later slice that may lose to the pruner. -func TestSetTxMinedStatus_RepairsRecoveredSubtreeBeforeMinedSet(t *testing.T) { +// TestRepairOvertakenTips_SetAsideSubtreeHoldsPruning: a subtree that cannot be read cannot +// be repaired, and the parents of its txs cannot be named. The hold stays while it is set +// aside, across new blocks, and is released only once the subtree reads and is repaired. +func TestRepairOvertakenTips_SetAsideSubtreeHoldsPruning(t *testing.T) { f := newOvertakeFixtureWithLaterSubtree(t) bv := f.newBlockValidation(t) - _ = f.fakeTipRepairClock(bv) + clock := f.fakeTipRepairClock(bv) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) blockC2 := f.overtakeB(t) - bv.subtreeStore = f.shortReads(f.subtreeStore, 0, tipRepairSubtreeReadAttempts*tipRepairMaxAttempts) + store := f.shortReads(f.subtreeStore, 0, 1<<30) + bv.subtreeStore = store settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) require.NotNil(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending, "precondition: subtree 0 is set aside") - require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: subtree 0's tx still looks mined") + require.NotZero(t, f.unminedSince(t, f.later), "precondition: the readable subtree is repaired") + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "a block with a subtree set aside is held") time.Sleep(10 * time.Millisecond) blockD2 := coinbaseOnlyBlock(t, blockC2.Hash(), f.nBits, 3, 0xd4) require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD2, "test", blockchainoptions.WithSubtreesSet(true))) + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockD2.Hash()), "a new block is set mined while the subtree stays unreadable") - var atMinedSet uint32 + settleTipRepair(t, bv, func() bool { return false }) + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "still held after a new block and more passes") - probe := &minedSetProbe{ClientI: f.blockchainClient, block: *blockD2.Hash(), at: func() { atMinedSet = f.unminedSince(t, f.bOnly) }} - bv.blockchainClient = probe + store.short.Store(0) + clock.advance(tipRepairPendingMaxBackoff) - require.NoError(t, bv.setTxMinedStatus(f.ctx, blockD2.Hash())) - require.Equal(t, int32(1), probe.calls.Load(), "precondition: D' was set mined") - require.NotZero(t, atMinedSet, "the recovered subtree's tx is re-marked before D' gets mined_set") + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "the subtree is repaired once it reads") + require.Empty(t, f.heldBlocks(t), "and the hold is released") } // cappedHeadersClient rejects header and block-summary requests over the configured caps, @@ -2264,29 +2295,70 @@ func TestRepairOvertakenTips_SeedPagesForkBlocksWithinCap(t *testing.T) { require.True(t, tracked, "fork block Y, below a full page at height 2, is seeded") } -// TestSetTxMinedStatus_FailedRepairLeavesMinedSetUnset: when the repair keeps failing, the -// block is not set mined, so the pruner cannot process its height. The error is the -// repair's, not the block's: a tracked block that is gone must not make the worker take the -// block it is setting mined for gone too and drop its retry. -func TestSetTxMinedStatus_FailedRepairLeavesMinedSetUnset(t *testing.T) { +// TestSetTxMinedStatus_FailingRepairDoesNotStopHealthyBlocks: a repair that keeps failing +// holds pruning, not the chain. A healthy new block is still set mined, so its children pass +// the parent-mined check, while the overtaken block stays held. +func TestSetTxMinedStatus_FailingRepairDoesNotStopHealthyBlocks(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) + + // Every attempt to re-mark B's txs fails, as a store fault that does not clear would. + f.store.failMarkFalse.Store(1 << 30) + + settleTipRepair(t, bv, func() bool { return false }) + require.Zero(t, f.unminedSince(t, f.bOnly), "precondition: B's repair keeps failing") + + time.Sleep(10 * time.Millisecond) + + blockD2 := coinbaseOnlyBlock(t, blockC2.Hash(), f.nBits, 3, 0xd4) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD2, "test", blockchainoptions.WithSubtreesSet(true))) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, blockD2.Hash())) + + _, meta, err := f.blockchainClient.GetBlockHeader(f.ctx, blockD2.Hash()) + require.NoError(t, err) + require.True(t, meta.MinedSet, "the healthy block is set mined") + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "B stays held while its repair fails") +} + +// TestRepairOvertakenTips_HoldSurvivesRestart: the blocks held when the process stops are read +// back from the hold when it starts again, so their repair finishes even when they are too +// deep below the tip for the seed to find, and the hold is not released before. +func TestRepairOvertakenTips_HoldSurvivesRestart(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + blockC2 := f.overtakeB(t) - // A tracked block the blockchain no longer has: every attempt to repair it fails. - bv.minedOnChain.track(chainhash.HashH([]byte("gone")), 999, 1) + f.store.failMarkFalse.Store(1 << 30) + settleTipRepair(t, bv, func() bool { return false }) + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "precondition: B is held") + // The winning chain grows past the seed's fork depth while the process is down. time.Sleep(10 * time.Millisecond) - blockD := coinbaseOnlyBlock(t, f.blockB.Hash(), f.nBits, 2, 0xd0) - require.NoError(t, f.blockchainClient.AddBlock(f.ctx, blockD, "test", blockchainoptions.WithSubtreesSet(true))) + tip := blockC2 + for height := uint32(3); height <= 3+tipRepairSeedForkDepth; height++ { + next := coinbaseOnlyBlock(t, tip.Hash(), f.nBits, height, 0xe0) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, next, "test", blockchainoptions.WithSubtreesSet(true))) + tip = next + } - err := bv.setTxMinedStatus(f.ctx, blockD.Hash()) - require.Error(t, err) - require.False(t, errors.Is(err, errors.ErrBlockNotFound), "the error is not the block's own: %v", err) + f.store.failMarkFalse.Store(0) + + restarted := f.newBlockValidation(t) + require.NoError(t, restarted.setTxMinedStatus(f.ctx, tip.Hash())) + + _, tracked := restarted.minedOnChain.snapshot()[*f.blockB.Hash()] + require.True(t, tracked, "B is tracked again from the hold") + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "the hold is not released before B is repaired") - _, meta, err := f.blockchainClient.GetBlockHeader(f.ctx, blockD.Hash()) + _, err := drainTipRepair(t, restarted) require.NoError(t, err) - require.False(t, meta.MinedSet, "mined_set stays unset while the repair fails") + require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired after the restart") + require.Empty(t, f.heldBlocks(t), "and the hold is released") } diff --git a/services/pruner/metrics.go b/services/pruner/metrics.go index 767b3d7b50..74a55ad919 100644 --- a/services/pruner/metrics.go +++ b/services/pruner/metrics.go @@ -20,6 +20,9 @@ var ( // prunerPreserveParentsFailing counts consecutive cycles whose Phase 1 failed; Phase 2 // deletes nothing while it is above zero, so alert when it stays up prunerPreserveParentsFailing prometheus.Gauge + // prunerOvertakenTipRepairHeldBlocks is the number of overtaken blocks block validation + // holds pruning for; nothing is deleted while above zero. + prunerOvertakenTipRepairHeldBlocks prometheus.Gauge // Blob deletion metrics blobDeletionScheduledTotal *prometheus.CounterVec @@ -114,6 +117,15 @@ func _initPrometheusMetrics() { }, ) + prunerOvertakenTipRepairHeldBlocks = promauto.NewGauge( + prometheus.GaugeOpts{ + Namespace: "teranode", + Subsystem: "pruner", + Name: "overtaken_tip_repair_held_blocks", + Help: "Overtaken blocks whose mined-status repair block validation has not finished; nothing is deleted while above zero. Stuck above zero means a block's subtree cannot be read or its repair keeps failing", + }, + ) + prunerPreserveParentsFailing = promauto.NewGauge( prometheus.GaugeOpts{ Namespace: "teranode", diff --git a/services/pruner/worker.go b/services/pruner/worker.go index b660911eee..b3a71919cd 100644 --- a/services/pruner/worker.go +++ b/services/pruner/worker.go @@ -2,11 +2,14 @@ package pruner import ( "context" + "database/sql" "fmt" + "strings" "time" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/model" "github.com/bsv-blockchain/teranode/services/blockchain" "github.com/bsv-blockchain/teranode/stores/utxo" "github.com/bsv-blockchain/teranode/util/retry" @@ -69,6 +72,46 @@ func (s *Server) checkBlockAssemblySafeForPruner(ctx context.Context, phase stri return true } +// overtakenTipRepairHeld reports whether block validation holds pruning for an overtaken +// block whose repair has not finished (model.OvertakenTipRepairHoldKey). No record means no +// hold. A record that cannot be read may be holding, so it counts as held. +func (s *Server) overtakenTipRepairHeld(ctx context.Context, blockHashStr string, blockHeight uint32) bool { + if s.blockchainClient == nil { + return false + } + + data, err := s.blockchainClient.GetState(ctx, model.OvertakenTipRepairHoldKey) + if err != nil { + if errors.Is(err, errors.ErrNotFound) || strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + prunerOvertakenTipRepairHeldBlocks.Set(0) + return false + } + + s.logger.Errorf("[pruner][%s:%d] failed to read the overtaken-tip repair hold, deleting nothing this cycle: %v", blockHashStr, blockHeight, err) + prunerErrors.WithLabelValues("overtaken_tip_repair_hold").Inc() + + return true + } + + held, err := model.DecodeOvertakenTipRepairHold(data) + if err != nil { + s.logger.Errorf("[pruner][%s:%d] unreadable overtaken-tip repair hold, deleting nothing this cycle: %v", blockHashStr, blockHeight, err) + prunerErrors.WithLabelValues("overtaken_tip_repair_hold").Inc() + + return true + } + + prunerOvertakenTipRepairHeldBlocks.Set(float64(len(held))) + + if len(held) == 0 { + return false + } + + s.logger.Warnf("[pruner][%s:%d] deleting nothing: block validation is still repairing %d overtaken blocks, lowest %s at height %d", blockHashStr, blockHeight, len(held), held[0].Hash.String(), held[0].Height) + + return true +} + // waitForBlockMinedStatus waits for the block to have mined_set=true, indicating that // block validation has completed. Returns true if mined, false otherwise. // This function will retry checking the mined status until the configured timeout is reached, @@ -199,15 +242,24 @@ func (s *Server) prunerProcessor(ctx context.Context) { continue } + // While block validation is repairing an overtaken block, some transactions may look + // mined while off the longest chain, and Phase 1 cannot preserve their parents. + // Nothing is deleted until the repair finishes: no blob deletion, which would take + // the overtaken block's subtrees the repair reads, and no Phase 2. + held := s.overtakenTipRepairHeld(ctx, blockHashStr, blockHeight) + // Safety check passed - wake blob deletion worker to run concurrently with Phase 1&2. // It works off the blockchain service's queued blob batches, independent of parent - // preservation, so it runs even in a cycle whose Phase 1 fails. - select { - case <-s.blobNotify: - default: + // preservation, so it runs even in a cycle whose Phase 1 fails. A held cycle skips + // it; the next notification deletes everything queued up to its height. + if !held { + select { + case <-s.blobNotify: + default: + } + s.blobNotify <- sig + s.logger.Debugf("[pruner][%s:%d] notified blob deletion worker", blockHashStr, blockHeight) } - s.blobNotify <- sig - s.logger.Debugf("[pruner][%s:%d] notified blob deletion worker", blockHashStr, blockHeight) prunerActive.Set(1) @@ -277,6 +329,14 @@ func (s *Server) prunerProcessor(ctx context.Context) { continue } + if held { + prunerSkipped.WithLabelValues("overtaken_tip_repair_hold").Inc() + prunerActive.Set(0) + + // Do not update lastProcessedHeight: the next notification must be processed + continue + } + if s.prunerService != nil { startTime := time.Now() recordsProcessed, err := s.prunerService.Prune(ctx, blockHeight, blockHashStr) diff --git a/services/pruner/worker_test.go b/services/pruner/worker_test.go index 0a9b4ca581..5a57c27ec5 100644 --- a/services/pruner/worker_test.go +++ b/services/pruner/worker_test.go @@ -2,11 +2,14 @@ package pruner import ( "context" + "database/sql" + "sync/atomic" "testing" "time" "github.com/bsv-blockchain/go-bt/v2/chainhash" "github.com/bsv-blockchain/teranode/errors" + "github.com/bsv-blockchain/teranode/model" "github.com/bsv-blockchain/teranode/services/blockassembly" blockassembly_api "github.com/bsv-blockchain/teranode/services/blockassembly/blockassembly_api" "github.com/bsv-blockchain/teranode/services/blockchain" @@ -179,6 +182,8 @@ func TestWaitForBlockMinedStatusProceeds(t *testing.T) { logger := ulogger.New("test") blockchainMock := &blockchain.Mock{} + // No overtaken-tip repair hold. + blockchainMock.On("GetState", mock.Anything, model.OvertakenTipRepairHoldKey).Return([]byte(nil), sql.ErrNoRows).Maybe() blockAssemblyMock := &blockassembly.Mock{} server := &Server{ @@ -234,6 +239,8 @@ func TestWaitForBlockMinedStatusTimeout(t *testing.T) { logger := ulogger.New("test") blockchainMock := &blockchain.Mock{} + // No overtaken-tip repair hold. + blockchainMock.On("GetState", mock.Anything, model.OvertakenTipRepairHoldKey).Return([]byte(nil), sql.ErrNoRows).Maybe() blockAssemblyMock := &blockassembly.Mock{} server := &Server{ @@ -333,6 +340,8 @@ func testSkipDuringCatchupSkipsInState(t *testing.T, state blockchain.FSMStateTy logger := ulogger.New("test") blockchainMock := &blockchain.Mock{} + // No overtaken-tip repair hold. + blockchainMock.On("GetState", mock.Anything, model.OvertakenTipRepairHoldKey).Return([]byte(nil), sql.ErrNoRows).Maybe() blockchainMock.On("GetFSMCurrentState", mock.Anything).Return(&state, nil) server := &Server{ @@ -383,6 +392,8 @@ func TestSkipDuringCatchupProceedsWhenRunning(t *testing.T) { logger := ulogger.New("test") blockchainMock := &blockchain.Mock{} + // No overtaken-tip repair hold. + blockchainMock.On("GetState", mock.Anything, model.OvertakenTipRepairHoldKey).Return([]byte(nil), sql.ErrNoRows).Maybe() running := blockchain.FSMStateRUNNING blockchainMock.On("GetFSMCurrentState", mock.Anything).Return(&running, nil) @@ -462,6 +473,8 @@ func TestStart_FSMContextCancellation(t *testing.T) { ctx := context.Background() mockBlockchainClient := &blockchain.Mock{} + // No overtaken-tip repair hold. + mockBlockchainClient.On("GetState", mock.Anything, model.OvertakenTipRepairHoldKey).Return([]byte(nil), sql.ErrNoRows).Maybe() mockBlockchainClient.On("WaitUntilFSMTransitionFromIdleState", mock.Anything).Return(context.Canceled) server := &Server{ @@ -763,3 +776,131 @@ func TestSkipPreserveParentsStillPrunes(t *testing.T) { mockStore.AssertNotCalled(t, "GetPrunableUnminedTxIterator", mock.Anything) } + +// repairHoldClient answers GetState for the overtaken-tip repair hold with whatever hold +// holds; the pruner makes no other blockchain call in these tests. +type repairHoldClient struct { + blockchain.ClientI + hold atomic.Pointer[repairHoldAnswer] +} + +type repairHoldAnswer struct { + data []byte + err error +} + +func (c *repairHoldClient) GetState(_ context.Context, key string) ([]byte, error) { + if key != model.OvertakenTipRepairHoldKey { + return nil, errors.NewProcessingError("unexpected state key %s", key) + } + + a := c.hold.Load() + + return a.data, a.err +} + +func newRepairHoldTestServer(ctx context.Context, client *repairHoldClient, pruneSvc *recordingPruner) *Server { + mockStore := &utxo.MockUtxostore{} + mockStore.On("GetPrunableUnminedTxIterator", mock.Anything).Return(emptyIterator{}, nil) + mockStore.On("ProcessExpiredPreservations", mock.Anything, mock.Anything).Return(nil) + + server := newPhase1TestServer(ctx, mockStore, pruneSvc, false) + server.blockchainClient = client + + return server +} + +// While block validation holds pruning for an overtaken block whose repair has not +// finished, the pruner deletes nothing: no DAH deletion and no blob deletion, and the height +// is not advanced. Once the hold is released, the next block prunes as usual. +func TestRepairHoldSkipsDeletion(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + held := model.EncodeOvertakenTipRepairHold([]model.HeldBlock{{Hash: chainhash.HashH([]byte("b")), ID: 7, Height: 90}}) + + client := &repairHoldClient{} + client.hold.Store(&repairHoldAnswer{data: held}) + + pruneSvc := &recordingPruner{calls: make(chan uint32, 2)} + server := newRepairHoldTestServer(ctx, client, pruneSvc) + + skippedBefore := getCounterValue(t, prunerSkipped, "overtaken_tip_repair_hold") + + go server.prunerProcessor(ctx) + server.pruneNotify <- pruneSignal{blockHeight: 101, blockHash: chainhash.Hash{}} + + require.Eventually(t, func() bool { + return getCounterValue(t, prunerSkipped, "overtaken_tip_repair_hold") == skippedBefore+1 + }, 2*time.Second, 10*time.Millisecond, "the cycle is counted as held") + + require.Empty(t, pruneSvc.calls, "no DAH deletion while held") + require.Empty(t, server.blobNotify, "no blob deletion while held") + require.Equal(t, uint32(0), server.lastProcessedHeight.Load(), "the height is retried") + require.Equal(t, float64(1), testutil.ToFloat64(prunerOvertakenTipRepairHeldBlocks)) + + client.hold.Store(&repairHoldAnswer{data: model.EncodeOvertakenTipRepairHold(nil)}) + server.pruneNotify <- pruneSignal{blockHeight: 102, blockHash: chainhash.Hash{}} + + select { + case h := <-pruneSvc.calls: + require.Equal(t, uint32(102), h) + case <-time.After(2 * time.Second): + t.Fatal("pruning did not resume once the hold was released") + } + + require.Equal(t, float64(0), testutil.ToFloat64(prunerOvertakenTipRepairHeldBlocks)) +} + +// A node whose block validation has never held pruning has no hold record: that is not a +// hold. +func TestRepairHoldMissingIsNotHeld(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := &repairHoldClient{} + client.hold.Store(&repairHoldAnswer{err: errors.NewStorageError("no hold", sql.ErrNoRows)}) + + pruneSvc := &recordingPruner{calls: make(chan uint32, 2)} + server := newRepairHoldTestServer(ctx, client, pruneSvc) + + go server.prunerProcessor(ctx) + server.pruneNotify <- pruneSignal{blockHeight: 101, blockHash: chainhash.Hash{}} + + select { + case h := <-pruneSvc.calls: + require.Equal(t, uint32(101), h) + case <-time.After(2 * time.Second): + t.Fatal("a missing hold record held pruning") + } +} + +// A hold that cannot be read may be holding: the cycle deletes nothing and is retried. +func TestRepairHoldReadErrorHolds(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := &repairHoldClient{} + client.hold.Store(&repairHoldAnswer{err: errors.NewServiceError("blockchain unavailable")}) + + pruneSvc := &recordingPruner{calls: make(chan uint32, 2)} + server := newRepairHoldTestServer(ctx, client, pruneSvc) + + errsBefore := getCounterValue(t, prunerErrors, "overtaken_tip_repair_hold") + + go server.prunerProcessor(ctx) + server.pruneNotify <- pruneSignal{blockHeight: 101, blockHash: chainhash.Hash{}} + + require.Eventually(t, func() bool { + return getCounterValue(t, prunerErrors, "overtaken_tip_repair_hold") == errsBefore+1 + }, 2*time.Second, 10*time.Millisecond) + + require.Empty(t, pruneSvc.calls, "no DAH deletion when the hold cannot be read") + require.Equal(t, uint32(0), server.lastProcessedHeight.Load()) +} diff --git a/settings.conf b/settings.conf index afc632ec44..f3a7475787 100644 --- a/settings.conf +++ b/settings.conf @@ -645,9 +645,8 @@ blockvalidation_recentBlockIDsLimit.docker.m = 10000 blockvalidation_recentBlockIDsLimit.dev = 1000 # Transactions re-marked per step of the overtaken-tip repair, which runs on the setMined -# worker (issue 2051). Every repair due runs in full before a block's mined status is -# written, so the pruner cannot process that height first; between blocks, one slice is -# the most a step holds the worker. +# worker between blocks (issue 2051). Bounds how long one step delays the next block's +# mined status. The pruner deletes nothing until the repair has finished. blockvalidation_tipRepairSliceSize = 250000 # Skip the per-transaction UTXO lock/unlock during quick-validation catchup (default: false). diff --git a/settings/blockvalidation_settings.go b/settings/blockvalidation_settings.go index 355c4908e6..e529d25274 100644 --- a/settings/blockvalidation_settings.go +++ b/settings/blockvalidation_settings.go @@ -73,7 +73,7 @@ type BlockValidationSettings struct { MaxBlocksBehindBlockAssembly int `key:"blockvalidation_maxBlocksBehindBlockAssembly" desc:"Maximum blocks behind Block Assembly" default:"20" category:"BlockValidation" usage:"Sync tolerance" type:"int" longdesc:"### Purpose\nSets the maximum blocks Block Validation can lag behind Block Assembly.\n\n### How It Works\n- Block Assembly may produce blocks faster than Block Validation can process\n- When this threshold is exceeded, catchup mode or waiting is triggered\n- Prevents unbounded queue growth between services\n\n### Recommendations\n- **20** (default) - Reasonable buffer for processing delays\n- Increase for systems with variable validation times\n- Decrease to maintain tighter synchronization"` PeriodicProcessingInterval time.Duration `key:"blockvalidation_periodic_processing_interval" desc:"Interval for periodic processing" default:"1m" category:"BlockValidation" usage:"Background task frequency" type:"duration" longdesc:"### Purpose\nSets the interval for periodic background processing tasks.\n\n### Tasks Performed\n- Checking mined status of pending blocks\n- Cache cleanup and maintenance\n- Resource cleanup and garbage collection\n\n### Recommendations\n- **1m** (default) - Regular maintenance without excessive overhead\n- Decrease for faster status updates (more CPU usage)\n- Increase for reduced background activity"` RecentBlockIDsLimit uint64 `key:"blockvalidation_recentBlockIDsLimit" desc:"Maximum recent block IDs to load" default:"50000" category:"BlockValidation" usage:"Memory limit for recent block IDs" type:"uint64" longdesc:"### Purpose\nLimits the number of recent block IDs loaded into memory for double-spend checking.\n\n### How It Works\nRecent block IDs are kept in memory to support fast double-spend detection. This setting prevents unbounded memory growth.\n\n### Recommendations\n- **50000** (default) - Covers typical scenarios\n- Increase for deeper history requirements\n- Decrease for memory-constrained environments"` - TipRepairSliceSize int `key:"blockvalidation_tipRepairSliceSize" desc:"Transactions re-marked per overtaken-tip repair slice" default:"250000" category:"BlockValidation" usage:"Bounds the batch each repair step reads and writes" type:"int" longdesc:"### Purpose\nBounds the work of one slice of the overtaken-tip repair.\n\n### How It Works\nWhen a block whose transactions were marked mined is overtaken before block assembly moved onto it, block validation re-marks the transactions that are not mined on the new chain. It does this on the setMined worker a slice of transactions at a time. Every repair due runs in full before the next block's mined status is written, so the pruner cannot process that height first: after a reorg, the next block's mined status waits for the whole repair, whatever the slice size. Between blocks, one slice is the most a repair step holds the worker.\n\n### Recommendations\n- **250000** (default)\n- Decrease to lower the memory one step uses"` + TipRepairSliceSize int `key:"blockvalidation_tipRepairSliceSize" desc:"Transactions re-marked per overtaken-tip repair slice" default:"250000" category:"BlockValidation" usage:"Bounds how long one repair slice holds the setMined worker" type:"int" longdesc:"### Purpose\nBounds the work of one slice of the overtaken-tip repair.\n\n### How It Works\nWhen a block whose transactions were marked mined is overtaken before block assembly moved onto it, block validation re-marks the transactions that are not mined on the new chain. It does this on the setMined worker, between blocks, a slice of transactions at a time, so the next block's mined status is delayed by at most one slice. The pruner deletes nothing until the repair has finished.\n\n### Recommendations\n- **250000** (default)\n- Decrease if setTxMined latency matters more than repair speed"` // Catchup configuration CatchupChBufferSize int `key:"blockvalidation_catchupCh_buffer_size" desc:"Buffer size for catchup channel" default:"100" category:"BlockValidation" usage:"Queue for catchup blocks" type:"int" longdesc:"### Purpose\nSets the buffer size for the channel receiving catchup block notifications.\n\n### How It Works\nDuring catchup mode, blocks are queued for processing. This buffer prevents blocking when multiple blocks arrive quickly.\n\n### Recommendations\n- **100** (default) - Handles typical catchup scenarios\n- Increase if experiencing backpressure during large syncs"` UseCatchupWhenBehind bool `key:"blockvalidation_useCatchupWhenBehind" desc:"Enable automatic catchup mode" default:"false" category:"BlockValidation" usage:"Auto-switch to catchup" type:"bool" longdesc:"### Purpose\nAutomatically switches to optimized catchup mode when the node falls behind the network.\n\n### How It Works\n- Monitors chain height relative to known network tip\n- When behind by threshold, switches to catchup mode\n- Catchup uses batch fetching and quick validation for faster sync\n- Returns to normal mode once caught up\n\n### Values\n- **false** (default) - Always use normal validation\n- **true** - Enable automatic catchup mode switching\n\n### Recommendations\n- Enable for nodes that may experience periods of being offline\n- Keep disabled for always-connected mining nodes"` diff --git a/stores/utxo/sql/parent_outputs.go b/stores/utxo/sql/parent_outputs.go index 9c704b5df5..4c6e91705a 100644 --- a/stores/utxo/sql/parent_outputs.go +++ b/stores/utxo/sql/parent_outputs.go @@ -124,6 +124,12 @@ func (s *Store) parentOutputsChunkWithRetry(ctx context.Context, chunk []outpoin // every outpoint whose transaction exists. Outpoints with no transaction row are // absent from the map. func (s *Store) parentOutputsChunk(ctx context.Context, chunk []outpointPair) (map[utxo.Outpoint]utxo.ParentOutput, error) { + if s.parentOutputsReadHook != nil { + if err := s.parentOutputsReadHook(); err != nil { + return nil, err + } + } + valuesClause, args := buildCompositeValuesPairs(chunk, 1, s.engine) // The CTE form works on both Postgres and SQLite (see PreviousOutputsDecorate). diff --git a/stores/utxo/sql/parent_outputs_test.go b/stores/utxo/sql/parent_outputs_test.go index 7f8a89ba38..dc265a92dd 100644 --- a/stores/utxo/sql/parent_outputs_test.go +++ b/stores/utxo/sql/parent_outputs_test.go @@ -140,3 +140,31 @@ func TestParentOutputsForValidationRetriesLockError(t *testing.T) { require.Equal(t, 1, attempts) }) } + +// A lock error on a parent-output chunk is retried by ParentOutputsForValidation itself, not +// returned as a per-outpoint fault. The hook refuses the chunk's first read with the error +// SQLite's shared cache raises for a lock cycle; without the retry at the call site every +// outpoint of the chunk comes back with that error. +func TestParentOutputsForValidationRetriesLockErrorFromChunk(t *testing.T) { + ctx := context.Background() + db, tx := setup(ctx, t) + + _, err := db.Create(ctx, tx, 0) + require.NoError(t, err) + + refusals := 1 + db.parentOutputsReadHook = func() error { + if refusals > 0 { + refusals-- + return errors.NewStorageError("database table is locked: database is deadlocked (6)") + } + + return nil + } + + answers, err := db.ParentOutputsForValidation(ctx, []utxo.Outpoint{{TxID: *tx.TxIDChainHash(), Vout: 0}}) + require.NoError(t, err) + require.Len(t, answers, 1) + require.NoError(t, answers[0].Err, "the refused read is retried") + require.Zero(t, refusals, "precondition: the first read was refused") +} diff --git a/stores/utxo/sql/sql.go b/stores/utxo/sql/sql.go index 98e609e564..274ffde61e 100644 --- a/stores/utxo/sql/sql.go +++ b/stores/utxo/sql/sql.go @@ -102,16 +102,19 @@ type batchSpend struct { // Store implements the UTXO store interface using a SQL database backend. type Store struct { - logger ulogger.Logger - settings *settings.Settings - db *usql.DB - storeURL *url.URL - engine string - ctx context.Context - spendBatcher *batcher.Batcher[batchSpend] - getBatcher *batcher.Batcher[batchGetItem] - createBatcher *batcher.Batcher[batchCreateItem] - unlockBatcher *batcher.Batcher[batchUnlockItem] + logger ulogger.Logger + // parentOutputsReadHook, when set by a test, runs before each parent-output chunk read + // and fails the read with its error. Always nil in production. + parentOutputsReadHook func() error + settings *settings.Settings + db *usql.DB + storeURL *url.URL + engine string + ctx context.Context + spendBatcher *batcher.Batcher[batchSpend] + getBatcher *batcher.Batcher[batchGetItem] + createBatcher *batcher.Batcher[batchCreateItem] + unlockBatcher *batcher.Batcher[batchUnlockItem] // utxo.BlockStateFields supplies the chain-tip height and median block time // as one atomic snapshot, and with them the Store interface's six From 9e2f02ca77c927f8acceef0353ef3c3036e2c64b Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 16:53:54 +0100 Subject: [PATCH 29/33] fix(pruner): hold for a gone subtree too, re-check the hold before phase 2 and in the blob worker From the interleaving review of the pruning hold: - A subtree of an overtaken block that is gone from the store was skipped and the block counted as repaired, releasing the hold although the txs in it may still look mined. It is now set aside and held like one that cannot be read. - The pruner read the hold once, before phase 1, so a hold written while phase 1 ran did not stop that cycle's phase 2. It reads it again just before deleting. - A blob deletion signal queued before a hold was written still ran. The blob deletion worker now checks the hold itself. - After a restart, a recorded hold was only read back once a block was written on the chain. Any setTxMined now reads it back. The blockchain MockStore keeps state instead of panicking on it. --- CHANGELOG.md | 6 +- services/blockvalidation/BlockValidation.go | 3 + .../blockvalidation/overtaken_tip_repair.go | 45 ++++++++--- .../overtaken_tip_repair_test.go | 74 +++++++++++++++++-- services/pruner/blob_deletion_test.go | 17 +++++ services/pruner/blob_deletion_worker.go | 7 ++ services/pruner/worker.go | 5 ++ services/pruner/worker_test.go | 70 +++++++++++++++++- stores/blockchain/mock.go | 25 ++++++- stores/blockchain/mock_test.go | 16 ++++ 10 files changed, 245 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aaa78ec49f..f4b52dc036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,9 +116,9 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea key `OvertakenTipRepairHold`, before it sets the mined status of the block that overtook them, and the pruner skips UTXO deletion (phase 2) and blob deletion while any is listed; parent preservation (phase 1) still runs, and deletion resumes on its own - once the repair finishes. A subtree of an overtaken block that cannot be read is read - again on every block and holds pruning until it reads or the block leaves the - 1000-block repair window. `teranode_pruner_overtaken_tip_repair_held_blocks` and + once the repair finishes. A subtree of an overtaken block that cannot be read, or is + gone from the subtree store, is tried again on every block and holds pruning until it + reads or the block leaves the 1000-block repair window. `teranode_pruner_overtaken_tip_repair_held_blocks` and `teranode_blockvalidation_tip_repair_subtrees_set_aside` show a hold; a hold that stays above zero means a subtree file is corrupt or a store keeps failing, and disk use grows until it is resolved. A repair that keeps failing no longer stops healthy blocks: their diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index def3074cb3..912bd33c72 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -347,6 +347,9 @@ type BlockValidation struct { // is set (guarded by tipRepairMu). tipRepairHold []byte tipRepairHoldWritten bool + // tipRepairHoldChecked is set once a hold recorded before this process started has been + // looked for (guarded by tipRepairMu). + tipRepairHoldChecked bool // tipRepairRetry is set while the last slice failed, so the periodic tick retries it. tipRepairRetry atomic.Bool // tipRepairWake requests a slice when the earliest pass over subtrees set aside comes diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 4438af28c1..4c87846dd3 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -545,6 +545,23 @@ func (u *BlockValidation) seedMinedOnChainTrackerLocked(ctx context.Context, blo return nil } +// holdRecordedLocked reports whether a hold recorded before this process started lists any +// block. It reads the record once: after that, only this process writes it. +func (u *BlockValidation) holdRecordedLocked(ctx context.Context, blockchainClient blockchain.ClientI) (bool, error) { + if u.tipRepairHoldChecked { + return false, nil + } + + data, err := blockchainClient.GetState(ctx, model.OvertakenTipRepairHoldKey) + if err != nil && !errors.Is(err, errors.ErrNotFound) && !strings.Contains(err.Error(), sql.ErrNoRows.Error()) { + return false, errors.NewServiceError("[repairOvertakenTips] failed to read the pruning hold", err) + } + + u.tipRepairHoldChecked = true + + return len(data) > 0, nil +} + // seedHeldBlocksLocked tracks the blocks a hold recorded before the restart lists. No record // means a node that has never held pruning. func (u *BlockValidation) seedHeldBlocksLocked(ctx context.Context, blockchainClient blockchain.ClientI) error { @@ -684,13 +701,16 @@ func (u *BlockValidation) refreshRepairHold(ctx context.Context, blockchainClien // has tracked a block, when there is nothing to do. func (u *BlockValidation) tipRepairHousekeepingLocked(ctx context.Context, blockchainClient blockchain.ClientI) (tracked map[chainhash.Hash]minedOnChainBlock, onMainChain map[uint32]bool, bestHeight uint32, active bool, err error) { - // Nothing to do until this process has written a block as on the longest chain. The - // restart seed waits for that too: after a restart the next block's setTxMined comes - // within one block interval, and a process that never marks blocks mined (or a test - // harness that never runs setTxMined) makes no blockchain calls here at all. A hold - // recorded before the restart stays as it is until then. + // Nothing to do until this process has written a block as on the longest chain, unless a + // hold recorded before a restart lists blocks still to repair: the restart seed waits for + // one or the other. After a restart the next block's setTxMined comes within one block + // interval, and a process that never marks blocks mined (or a test harness that never runs + // setTxMined) makes one state read here at most. if !u.tipRepairSeeded && u.minedOnChain.empty() { - return nil, nil, 0, false, nil + held, err := u.holdRecordedLocked(ctx, blockchainClient) + if err != nil || !held { + return nil, nil, 0, false, err + } } if !u.tipRepairSeeded { @@ -925,10 +945,10 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone -// is skipped with an error log. One that still cannot be loaded after tipRepairMaxAttempts -// failed loads, whatever the error, is set aside, so the rest of the block is repaired: a -// run of short reads looks the same as a corrupt file, and a store that stays down must not -// hold up the later subtrees. The block's next unmark job reads the set-aside subtrees +// is set aside at once. One that still cannot be loaded after tipRepairMaxAttempts failed +// loads, whatever the error, is set aside too, so the rest of the block is repaired: a run +// of short reads looks the same as a corrupt file, and a store that stays down must not hold +// up the later subtrees. A block with subtrees set aside is not done, so it holds pruning. The block's next unmark job reads the set-aside subtrees // again; a remine job with subtrees set aside leaves the block stamped, so a later remine // does them (see finishJob). Until a subtree is set aside, a load error fails the step, to // be retried. @@ -947,7 +967,10 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha switch { case isPermanentRepairError(err): - u.logger.Errorf("[repairOvertakenTips][%s] skipping subtree %s, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) + // Gone is no more readable than corrupt: its txs cannot be named, so the block + // stays held until it reads or leaves the repair window. + u.logger.Errorf("[repairOvertakenTips][%s] subtree %s is gone, setting it aside and holding pruning, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) + job.unreadable = append(job.unreadable, subtreeHash) case ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) job.unreadable = append(job.unreadable, subtreeHash) diff --git a/services/blockvalidation/overtaken_tip_repair_test.go b/services/blockvalidation/overtaken_tip_repair_test.go index c683167b2d..881bbda5ff 100644 --- a/services/blockvalidation/overtaken_tip_repair_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -554,21 +554,29 @@ func (c *extraSubtreeClient) GetBlock(ctx context.Context, blockHash *chainhash. block.TransactionCount, block.SizeInBytes, block.Height, block.ID) } -// TestRepairOvertakenTips_SkipsMissingSubtree: a subtree that is gone must not stop the -// repair of the block's other subtrees, nor keep the block retrying. -func TestRepairOvertakenTips_SkipsMissingSubtree(t *testing.T) { +// TestRepairOvertakenTips_MissingSubtreeSetAside: a subtree that is gone must not stop the +// repair of the block's other subtrees. Its txs cannot be named, so it is set aside and the +// block is not done: it keeps holding pruning, and its next pass waits on the back-off. +func TestRepairOvertakenTips_MissingSubtreeSetAside(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) _ = f.overtakeB(t) - bv.blockchainClient = &extraSubtreeClient{ClientI: f.blockchainClient} + client := &extraSubtreeClient{ClientI: f.blockchainClient} + bv.blockchainClient = client _, err := drainTipRepair(t, bv) require.NoError(t, err) require.NotZero(t, f.unminedSince(t, f.bOnly), "the readable subtree is repaired") - require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "and the block is done, not retried") + + missing, err := client.GetBlock(f.ctx, f.blockB.Hash()) + require.NoError(t, err) + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, b.done, "the block is not done while a subtree is gone") + require.Equal(t, []*chainhash.Hash{missing.Subtrees[0]}, b.pending, "the gone subtree is set aside") } // failingGetBlockClient fails GetBlock while fail is set: a blockchain service that is @@ -2362,3 +2370,59 @@ func TestRepairOvertakenTips_HoldSurvivesRestart(t *testing.T) { require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired after the restart") require.Empty(t, f.heldBlocks(t), "and the hold is released") } + +// TestRepairOvertakenTips_GoneSubtreeHoldsPruning: a subtree of an overtaken block that is +// gone from the store cannot be read any more than a corrupt one, so its txs, which may look +// mined while off the chain, cannot be named and their parents cannot be protected. The +// block is not done, and the pruning hold keeps it, instead of skipping the subtree. +func TestRepairOvertakenTips_GoneSubtreeHoldsPruning(t *testing.T) { + f := newOvertakeFixtureWithLaterSubtree(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewNotFoundError("subtree is gone")) + + settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + + b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] + require.False(t, b.done, "a block with a gone subtree is not done") + require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending) + require.NotZero(t, f.unminedSince(t, f.later), "the readable subtree is repaired") + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "and the block holds pruning") +} + +// TestRepairOvertakenTips_HoldReadBackBeforeAnyOnChainWrite: after a restart, the blocks a +// hold recorded are read back on the first setTxMined, even when that block is off the chain +// and tracks nothing, so their repair, and the release of the hold, does not wait for the +// first block written on the chain. +func TestRepairOvertakenTips_HoldReadBackBeforeAnyOnChainWrite(t *testing.T) { + f := newOvertakeFixture(t) + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + _ = f.overtakeB(t) + + f.store.failMarkFalse.Store(1 << 30) + settleTipRepair(t, bv, func() bool { return false }) + require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "precondition: B is held") + + f.store.failMarkFalse.Store(0) + + time.Sleep(10 * time.Millisecond) + + fork := coinbaseOnlyBlock(t, f.tSettings.ChainCfgParams.GenesisHash, f.nBits, 1, 0xf7) + require.NoError(t, f.blockchainClient.AddBlock(f.ctx, fork, "test", blockchainoptions.WithSubtreesSet(true))) + + restarted := f.newBlockValidation(t) + require.NoError(t, restarted.setTxMinedStatus(f.ctx, fork.Hash())) + + _, tracked := restarted.minedOnChain.snapshot()[*f.blockB.Hash()] + require.True(t, tracked, "B is tracked again from the hold") + + _, err := drainTipRepair(t, restarted) + require.NoError(t, err) + require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired") + require.Empty(t, f.heldBlocks(t), "and the hold is released") +} diff --git a/services/pruner/blob_deletion_test.go b/services/pruner/blob_deletion_test.go index 5fc9d2f287..7081d5bd8a 100644 --- a/services/pruner/blob_deletion_test.go +++ b/services/pruner/blob_deletion_test.go @@ -3,6 +3,7 @@ package pruner import ( "context" "crypto/rand" + "database/sql" "fmt" "net/http/httptest" "net/url" @@ -43,6 +44,11 @@ func newMockBlockchainClient() *mockBlockchainClient { } } +// GetState answers as a node that has never held pruning for an overtaken-tip repair. +func (m *mockBlockchainClient) GetState(context.Context, string) ([]byte, error) { + return nil, sql.ErrNoRows +} + func (m *mockBlockchainClient) ScheduleBlobDeletion(ctx context.Context, blobKey []byte, fileType string, storeType storetypes.BlobStoreType, deleteAtHeight uint32) (int64, bool, error) { id := m.nextID m.nextID++ @@ -548,6 +554,17 @@ type blobDeletionServiceClient struct { service *blockchain.Blockchain } +// GetState goes through the real blockchain service, as the gRPC client does: no +// overtaken-tip repair hold has been written, so the pruner must read the key as missing. +func (c *blobDeletionServiceClient) GetState(ctx context.Context, key string) ([]byte, error) { + resp, err := c.service.GetState(ctx, &blockchain_api.GetStateRequest{Key: key}) + if err != nil { + return nil, errors.UnwrapGRPC(err) + } + + return resp.Data, nil +} + func (c *blobDeletionServiceClient) AcquireBlobDeletionBatch(ctx context.Context, height uint32, limit int, lockTimeoutSeconds int, excludeStoreTypes []storetypes.BlobStoreType) (string, []*blockchain_api.ScheduledDeletion, error) { excluded := make([]int32, len(excludeStoreTypes)) for i, storeType := range excludeStoreTypes { diff --git a/services/pruner/blob_deletion_worker.go b/services/pruner/blob_deletion_worker.go index 79bc0589aa..c61932ae0f 100644 --- a/services/pruner/blob_deletion_worker.go +++ b/services/pruner/blob_deletion_worker.go @@ -65,6 +65,13 @@ func (s *Server) processBlobDeletionsAtHeight(blockHeight uint32, blockHash chai return } + // An overtaken block's repair still needs its subtrees. The pruner does not signal while + // held, but a signal queued before the hold was written still arrives here; the next one + // after the hold is released deletes everything queued up to its height. + if s.overtakenTipRepairHeld(ctx, blockHashStr, blockHeight) { + return + } + // Apply safety window: only delete blobs that are safetyWindow blocks behind the triggering height safetyWindow := s.settings.Pruner.BlobDeletionSafetyWindow if safetyWindow > 0 { diff --git a/services/pruner/worker.go b/services/pruner/worker.go index b3a71919cd..c2492679c1 100644 --- a/services/pruner/worker.go +++ b/services/pruner/worker.go @@ -329,6 +329,11 @@ func (s *Server) prunerProcessor(ctx context.Context) { continue } + // Read again: block validation may have written a hold while phase 1 ran. + if !held { + held = s.overtakenTipRepairHeld(ctx, blockHashStr, blockHeight) + } + if held { prunerSkipped.WithLabelValues("overtaken_tip_repair_hold").Inc() prunerActive.Set(0) diff --git a/services/pruner/worker_test.go b/services/pruner/worker_test.go index 5a57c27ec5..9950214cf0 100644 --- a/services/pruner/worker_test.go +++ b/services/pruner/worker_test.go @@ -13,7 +13,9 @@ import ( "github.com/bsv-blockchain/teranode/services/blockassembly" blockassembly_api "github.com/bsv-blockchain/teranode/services/blockassembly/blockassembly_api" "github.com/bsv-blockchain/teranode/services/blockchain" + "github.com/bsv-blockchain/teranode/services/blockchain/blockchain_api" "github.com/bsv-blockchain/teranode/settings" + "github.com/bsv-blockchain/teranode/stores/blob/storetypes" "github.com/bsv-blockchain/teranode/stores/utxo" prunerstore "github.com/bsv-blockchain/teranode/stores/utxo/pruner" "github.com/bsv-blockchain/teranode/ulogger" @@ -778,10 +780,19 @@ func TestSkipPreserveParentsStillPrunes(t *testing.T) { } // repairHoldClient answers GetState for the overtaken-tip repair hold with whatever hold -// holds; the pruner makes no other blockchain call in these tests. +// holds, or with next after the first read when next is set; it records blob deletion batch +// requests and makes no other blockchain call. type repairHoldClient struct { blockchain.ClientI - hold atomic.Pointer[repairHoldAnswer] + hold atomic.Pointer[repairHoldAnswer] + next atomic.Pointer[repairHoldAnswer] + reads atomic.Int32 + acquired atomic.Int32 +} + +func (c *repairHoldClient) AcquireBlobDeletionBatch(context.Context, uint32, int, int, []storetypes.BlobStoreType) (string, []*blockchain_api.ScheduledDeletion, error) { + c.acquired.Add(1) + return "", nil, nil } type repairHoldAnswer struct { @@ -795,6 +806,9 @@ func (c *repairHoldClient) GetState(_ context.Context, key string) ([]byte, erro } a := c.hold.Load() + if c.reads.Add(1) > 1 && c.next.Load() != nil { + a = c.next.Load() + } return a.data, a.err } @@ -904,3 +918,55 @@ func TestRepairHoldReadErrorHolds(t *testing.T) { require.Empty(t, pruneSvc.calls, "no DAH deletion when the hold cannot be read") require.Equal(t, uint32(0), server.lastProcessedHeight.Load()) } + +// A hold written while a cycle's phase 1 runs must still stop that cycle's phase 2: the hold +// is read again just before deleting. +func TestRepairHoldReadAgainBeforePhase2(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := &repairHoldClient{} + client.hold.Store(&repairHoldAnswer{data: model.EncodeOvertakenTipRepairHold(nil)}) + client.next.Store(&repairHoldAnswer{data: model.EncodeOvertakenTipRepairHold([]model.HeldBlock{{Hash: chainhash.HashH([]byte("b")), ID: 7, Height: 90}})}) + + pruneSvc := &recordingPruner{calls: make(chan uint32, 2)} + server := newRepairHoldTestServer(ctx, client, pruneSvc) + + skippedBefore := getCounterValue(t, prunerSkipped, "overtaken_tip_repair_hold") + + go server.prunerProcessor(ctx) + server.pruneNotify <- pruneSignal{blockHeight: 101, blockHash: chainhash.Hash{}} + + require.Eventually(t, func() bool { + return getCounterValue(t, prunerSkipped, "overtaken_tip_repair_hold") == skippedBefore+1 + }, 2*time.Second, 10*time.Millisecond, "the cycle is held by the second read") + + require.Empty(t, pruneSvc.calls, "no DAH deletion") + require.Equal(t, uint32(0), server.lastProcessedHeight.Load()) +} + +// The blob deletion worker checks the hold itself: a signal queued before the hold was +// written, or a pass already under way, must not delete an overtaken block's subtrees while +// its repair still needs them. +func TestRepairHoldStopsBlobDeletion(t *testing.T) { + initPrometheusMetrics() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + client := &repairHoldClient{} + client.hold.Store(&repairHoldAnswer{data: model.EncodeOvertakenTipRepairHold([]model.HeldBlock{{Hash: chainhash.HashH([]byte("b")), ID: 7, Height: 90}})}) + + server := newRepairHoldTestServer(ctx, client, &recordingPruner{calls: make(chan uint32, 1)}) + server.settings.Pruner.BlobDeletionBatchSize = 10 + + server.processBlobDeletionsAtHeight(101, chainhash.Hash{}) + require.Zero(t, client.acquired.Load(), "no deletion batch is taken while held") + + client.hold.Store(&repairHoldAnswer{data: model.EncodeOvertakenTipRepairHold(nil)}) + + server.processBlobDeletionsAtHeight(102, chainhash.Hash{}) + require.NotZero(t, client.acquired.Load(), "deletion resumes once the hold is released") +} diff --git a/stores/blockchain/mock.go b/stores/blockchain/mock.go index 6c505b5180..41b53e41f2 100644 --- a/stores/blockchain/mock.go +++ b/stores/blockchain/mock.go @@ -50,6 +50,8 @@ type MockStore struct { blockIDReservations map[chainhash.Hash]uint64 // state tracks the current state of the mock store (e.g., IDLE) state string + // keyedState holds what SetState wrote, by key + keyedState map[string][]byte // MainChainFastPathDisabled, when true, makes MainChainBlockHashesByHeights // return ok=false so callers exercise the per-height walk fallback (test hook). MainChainFastPathDisabled bool @@ -742,12 +744,31 @@ func (m *MockStore) GetBlockHeaderIDs(ctx context.Context, blockHash *chainhash. return []uint32{}, nil } +// GetState returns what SetState wrote under key, or a not-found error. func (m *MockStore) GetState(ctx context.Context, key string) ([]byte, error) { - panic(implementMe) + m.mu.RLock() + defer m.mu.RUnlock() + + data, ok := m.keyedState[key] + if !ok { + return nil, errors.NewNotFoundError("no state for key %s", key) + } + + return data, nil } +// SetState stores data under key. func (m *MockStore) SetState(ctx context.Context, key string, data []byte) error { - panic(implementMe) + m.mu.Lock() + defer m.mu.Unlock() + + if m.keyedState == nil { + m.keyedState = map[string][]byte{} + } + + m.keyedState[key] = append([]byte(nil), data...) + + return nil } func (m *MockStore) GetBlockIsMined(_ context.Context, blockHash *chainhash.Hash) (bool, error) { diff --git a/stores/blockchain/mock_test.go b/stores/blockchain/mock_test.go index b383da7304..3bfa34b113 100644 --- a/stores/blockchain/mock_test.go +++ b/stores/blockchain/mock_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/bsv-blockchain/teranode/errors" "github.com/stretchr/testify/require" ) @@ -18,3 +19,18 @@ func TestMockStore_GetLastNBlocksNonPositiveN(t *testing.T) { require.Empty(t, infos) } } + +// TestMockStore_State: state reads what was written, and a key never written is not found, +// as in the SQL store. +func TestMockStore_State(t *testing.T) { + m := NewMockStore() + + _, err := m.GetState(context.Background(), "k") + require.ErrorIs(t, err, errors.ErrNotFound) + + require.NoError(t, m.SetState(context.Background(), "k", []byte{1, 2})) + + data, err := m.GetState(context.Background(), "k") + require.NoError(t, err) + require.Equal(t, []byte{1, 2}, data) +} From ec1f90ff6900efefd8dc90f10311d4669792622c Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 17:19:12 +0100 Subject: [PATCH 30/33] fix(blockvalidation): give up a subtree that is gone instead of holding pruning for it, count both kinds of give-up A subtree that is gone from the store can never be read again, and the store cannot list a block's txs without it, so holding pruning for it only delayed the same loss by up to the 1000-block window while disk grew. It is given up as a gone block already is, with an error log and teranode_blockvalidation_tip_repair_abandoned_total{kind="subtree"}; a gone block now counts under kind="block". A subtree that cannot be read, which may recover, still holds pruning. --- CHANGELOG.md | 9 +++-- services/blockvalidation/metrics.go | 11 ++++++ .../blockvalidation/overtaken_tip_repair.go | 18 +++++---- .../overtaken_tip_repair_test.go | 39 +++++++++---------- 4 files changed, 46 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4b52dc036..c77164e84d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,9 +116,12 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea key `OvertakenTipRepairHold`, before it sets the mined status of the block that overtook them, and the pruner skips UTXO deletion (phase 2) and blob deletion while any is listed; parent preservation (phase 1) still runs, and deletion resumes on its own - once the repair finishes. A subtree of an overtaken block that cannot be read, or is - gone from the subtree store, is tried again on every block and holds pruning until it - reads or the block leaves the 1000-block repair window. `teranode_pruner_overtaken_tip_repair_held_blocks` and + once the repair finishes. A subtree of an overtaken block that cannot be read is tried + again on every block and holds pruning until it reads or the block leaves the + 1000-block repair window. A subtree or block that is gone from its store can never be + read, so it is given up at once and counted in + `teranode_blockvalidation_tip_repair_abandoned_total`: its transactions may stay marked + mined while off the chain, and the pruner can then delete parents they need. `teranode_pruner_overtaken_tip_repair_held_blocks` and `teranode_blockvalidation_tip_repair_subtrees_set_aside` show a hold; a hold that stays above zero means a subtree file is corrupt or a store keeps failing, and disk use grows until it is resolved. A repair that keeps failing no longer stops healthy blocks: their diff --git a/services/blockvalidation/metrics.go b/services/blockvalidation/metrics.go index 503556beae..1f3821c1fc 100644 --- a/services/blockvalidation/metrics.go +++ b/services/blockvalidation/metrics.go @@ -93,6 +93,7 @@ var ( prometheusBlockValidationSetMinedDrops prometheus.Counter prometheusBlockValidationSetMinedEnqueueOverflow prometheus.Counter prometheusBlockValidationTipRepairSubtreesSetAside prometheus.Gauge + prometheusBlockValidationTipRepairAbandoned *prometheus.CounterVec // outpoint-only fast-path counter: incremented once per block when the // below-checkpoint outpoint-only path is active (setting on, height ≤ highest @@ -288,6 +289,16 @@ func _initPrometheusMetrics() { }, ) + prometheusBlockValidationTipRepairAbandoned = promauto.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "teranode", + Subsystem: "blockvalidation", + Name: "tip_repair_abandoned_total", + Help: "Overtaken blocks (kind=block) and their subtrees (kind=subtree) the overtaken-tip repair gave up because they are gone. Their txs may stay marked mined while off the longest chain, and the pruner can delete parents those txs need; nothing can name them without the data.", + }, + []string{"kind"}, + ) + prometheusBlockValidationQuickCommitAddBlock = promauto.NewHistogram( prometheus.HistogramOpts{ Namespace: "teranode", diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 4c87846dd3..808cc248a5 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -945,10 +945,11 @@ func (u *BlockValidation) startJob(ctx context.Context, blockchainClient blockch // jobStep writes the next slice of the job's txs, first loading the next subtree if the // current one is used up, and finishes the job with its last slice. A subtree that is gone -// is set aside at once. One that still cannot be loaded after tipRepairMaxAttempts failed -// loads, whatever the error, is set aside too, so the rest of the block is repaired: a run -// of short reads looks the same as a corrupt file, and a store that stays down must not hold -// up the later subtrees. A block with subtrees set aside is not done, so it holds pruning. The block's next unmark job reads the set-aside subtrees +// is given up and counted: nothing can name its txs again. One that still cannot be loaded +// after tipRepairMaxAttempts failed loads, whatever the error, is set aside, so the rest of +// the block is repaired: a run of short reads looks the same as a corrupt file, and a store +// that stays down must not hold up the later subtrees. A block with subtrees set aside is +// not done, so it holds pruning until they read. The block's next unmark job reads the set-aside subtrees // again; a remine job with subtrees set aside leaves the block stamped, so a later remine // does them (see finishJob). Until a subtree is set aside, a load error fails the step, to // be retried. @@ -967,10 +968,10 @@ func (u *BlockValidation) jobStep(ctx context.Context, blockchainClient blockcha switch { case isPermanentRepairError(err): - // Gone is no more readable than corrupt: its txs cannot be named, so the block - // stays held until it reads or leaves the repair window. - u.logger.Errorf("[repairOvertakenTips][%s] subtree %s is gone, setting it aside and holding pruning, its txs may still look mined: %v", job.hash.String(), subtreeHash.String(), err) - job.unreadable = append(job.unreadable, subtreeHash) + // Gone for good: nothing can name its txs again, so holding pruning for it would + // only delay the loss. It is given up, as a gone block is (repairFailed). + u.logger.Errorf("[repairOvertakenTips][%s] giving up subtree %s, it is gone; its txs may stay marked mined while off the longest chain: %v", job.hash.String(), subtreeHash.String(), err) + prometheusBlockValidationTipRepairAbandoned.WithLabelValues("subtree").Inc() case ctx.Err() == nil && job.subtreeFailures >= tipRepairMaxAttempts: u.logger.Errorf("[repairOvertakenTips][%s] setting subtree %s aside after %d failed loads, it is read again later: %v", job.hash.String(), subtreeHash.String(), job.subtreeFailures, err) job.unreadable = append(job.unreadable, subtreeHash) @@ -1134,6 +1135,7 @@ func (u *BlockValidation) repairFailed(hash chainhash.Hash, gen uint64, err erro if failures >= tipRepairMaxAttempts { u.logger.Errorf("[repairOvertakenTips][%s] giving up after %d failed attempts; its txs may still look mined: %v", hash.String(), tipRepairMaxAttempts, err) + prometheusBlockValidationTipRepairAbandoned.WithLabelValues("block").Inc() u.minedOnChain.drop(hash) if u.tipRepairJob != nil && u.tipRepairJob.hash == hash { diff --git a/services/blockvalidation/overtaken_tip_repair_test.go b/services/blockvalidation/overtaken_tip_repair_test.go index 881bbda5ff..f029102b05 100644 --- a/services/blockvalidation/overtaken_tip_repair_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -554,10 +554,9 @@ func (c *extraSubtreeClient) GetBlock(ctx context.Context, blockHash *chainhash. block.TransactionCount, block.SizeInBytes, block.Height, block.ID) } -// TestRepairOvertakenTips_MissingSubtreeSetAside: a subtree that is gone must not stop the -// repair of the block's other subtrees. Its txs cannot be named, so it is set aside and the -// block is not done: it keeps holding pruning, and its next pass waits on the back-off. -func TestRepairOvertakenTips_MissingSubtreeSetAside(t *testing.T) { +// TestRepairOvertakenTips_SkipsMissingSubtree: a subtree that is gone must not stop the +// repair of the block's other subtrees, nor keep the block retrying. +func TestRepairOvertakenTips_SkipsMissingSubtree(t *testing.T) { f := newOvertakeFixture(t) bv := f.newBlockValidation(t) @@ -571,12 +570,7 @@ func TestRepairOvertakenTips_MissingSubtreeSetAside(t *testing.T) { require.NoError(t, err) require.NotZero(t, f.unminedSince(t, f.bOnly), "the readable subtree is repaired") - missing, err := client.GetBlock(f.ctx, f.blockB.Hash()) - require.NoError(t, err) - - b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] - require.False(t, b.done, "the block is not done while a subtree is gone") - require.Equal(t, []*chainhash.Hash{missing.Subtrees[0]}, b.pending, "the gone subtree is set aside") + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "and the block is done, not retried") } // failingGetBlockClient fails GetBlock while fail is set: a blockchain service that is @@ -633,6 +627,8 @@ func TestRepairOvertakenTips_GivesUpAfterMaxAttempts(t *testing.T) { bv.blockchainClient = &blockGoneClient{ClientI: f.blockchainClient} + before := testutil.ToFloat64(prometheusBlockValidationTipRepairAbandoned.WithLabelValues("block")) + for i := 0; i < tipRepairMaxAttempts; i++ { _, err := drainTipRepair(t, bv) require.Error(t, err, "attempt %d", i+1) @@ -640,6 +636,7 @@ func TestRepairOvertakenTips_GivesUpAfterMaxAttempts(t *testing.T) { _, tracked := bv.minedOnChain.snapshot()[*f.blockB.Hash()] require.False(t, tracked, "the block is dropped after the last attempt") + require.InDelta(t, before+1, testutil.ToFloat64(prometheusBlockValidationTipRepairAbandoned.WithLabelValues("block")), 0, "and counted") _, err := drainTipRepair(t, bv) require.NoError(t, err, "and no longer retried") @@ -2371,11 +2368,11 @@ func TestRepairOvertakenTips_HoldSurvivesRestart(t *testing.T) { require.Empty(t, f.heldBlocks(t), "and the hold is released") } -// TestRepairOvertakenTips_GoneSubtreeHoldsPruning: a subtree of an overtaken block that is -// gone from the store cannot be read any more than a corrupt one, so its txs, which may look -// mined while off the chain, cannot be named and their parents cannot be protected. The -// block is not done, and the pruning hold keeps it, instead of skipping the subtree. -func TestRepairOvertakenTips_GoneSubtreeHoldsPruning(t *testing.T) { +// TestRepairOvertakenTips_GoneSubtreeGivenUp: a subtree of an overtaken block that is gone +// from the store can never be read again, so nothing can name its txs: holding pruning for it +// would only delay the same loss. It is given up as a gone block is, counted so operators see +// that some txs may stay marked mined, and the rest of the block is repaired and released. +func TestRepairOvertakenTips_GoneSubtreeGivenUp(t *testing.T) { f := newOvertakeFixtureWithLaterSubtree(t) bv := f.newBlockValidation(t) @@ -2384,13 +2381,15 @@ func TestRepairOvertakenTips_GoneSubtreeHoldsPruning(t *testing.T) { bv.subtreeStore = f.erroringReads(f.subtreeStore, 0, 1<<30, errors.NewNotFoundError("subtree is gone")) - settleTipRepair(t, bv, func() bool { return bv.minedOnChain.snapshot()[*f.blockB.Hash()].pending != nil }) + before := testutil.ToFloat64(prometheusBlockValidationTipRepairAbandoned.WithLabelValues("subtree")) - b := bv.minedOnChain.snapshot()[*f.blockB.Hash()] - require.False(t, b.done, "a block with a gone subtree is not done") - require.Equal(t, []*chainhash.Hash{f.blockB.Subtrees[0]}, b.pending) + _, err := drainTipRepair(t, bv) + require.NoError(t, err) + + require.True(t, bv.minedOnChain.snapshot()[*f.blockB.Hash()].done, "the block is done without the gone subtree") require.NotZero(t, f.unminedSince(t, f.later), "the readable subtree is repaired") - require.Equal(t, []chainhash.Hash{*f.blockB.Hash()}, f.heldBlocks(t), "and the block holds pruning") + require.Empty(t, f.heldBlocks(t), "a gone subtree does not hold pruning") + require.InDelta(t, before+1, testutil.ToFloat64(prometheusBlockValidationTipRepairAbandoned.WithLabelValues("subtree")), 0, "and is counted") } // TestRepairOvertakenTips_HoldReadBackBeforeAnyOnChainWrite: after a restart, the blocks a From f4f7b73fbe5aff8129ae12d4863ae666303b36e8 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 17:44:56 +0100 Subject: [PATCH 31/33] fix(model): never write an empty overtaken-tip repair hold, it reaches the SQL state table as NULL A hold listing no blocks encoded to zero bytes. Over the gRPC blockchain client empty bytes arrive as nil, and the SQL state table refuses a NULL (NOT NULL constraint failed: state.data), so every refresh of an empty hold failed, no block got mined_set, and the chain stalled (chainintegrity, legacy-sync). The tests wrote through LocalClient, which passes the empty slice through, so none saw it. The record now starts with a version byte, so it is never empty, and a record without it is rejected. TestSetTxMinedStatus_EmptyHoldSurvivesTheWire sends the write through a protobuf round trip; TestChainIntegrity3Nodes fails locally with the old encoding and passes with this one. --- model/overtaken_tip_repair_hold.go | 21 ++++++++-- model/overtaken_tip_repair_hold_test.go | 15 +++++++- .../blockvalidation/overtaken_tip_repair.go | 9 ++++- .../overtaken_tip_repair_test.go | 38 +++++++++++++++++++ 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/model/overtaken_tip_repair_hold.go b/model/overtaken_tip_repair_hold.go index deaab39446..65c4b815af 100644 --- a/model/overtaken_tip_repair_hold.go +++ b/model/overtaken_tip_repair_hold.go @@ -10,12 +10,18 @@ import ( // OvertakenTipRepairHoldKey is the blockchain state key under which block validation records // the overtaken blocks whose repair has not finished. While the record lists any, the pruner // deletes nothing: their transactions may look mined while off the longest chain, so the -// pruner cannot tell which parents they still need. An absent or empty record holds nothing. +// pruner cannot tell which parents they still need. An absent record, or one listing no +// blocks, holds nothing. const OvertakenTipRepairHoldKey = "OvertakenTipRepairHold" // heldBlockSize is the encoded size of one HeldBlock: hash, id, height. const heldBlockSize = chainhash.HashSize + 4 + 4 +// overtakenTipRepairHoldVersion is the first byte of every record. It also keeps a record with +// no blocks from being empty, which the gRPC client would send as nil and the SQL state table +// would refuse. +const overtakenTipRepairHoldVersion = 1 + // HeldBlock is an overtaken block whose repair has not finished. type HeldBlock struct { Hash chainhash.Hash @@ -23,10 +29,11 @@ type HeldBlock struct { Height uint32 } -// EncodeOvertakenTipRepairHold encodes held blocks for OvertakenTipRepairHoldKey. No blocks -// encode as an empty record. +// EncodeOvertakenTipRepairHold encodes held blocks for OvertakenTipRepairHoldKey: a version +// byte, then the blocks. func EncodeOvertakenTipRepairHold(held []HeldBlock) []byte { - data := make([]byte, 0, len(held)*heldBlockSize) + data := make([]byte, 1, 1+len(held)*heldBlockSize) + data[0] = overtakenTipRepairHoldVersion for _, b := range held { data = append(data, b.Hash[:]...) @@ -39,6 +46,12 @@ func EncodeOvertakenTipRepairHold(held []HeldBlock) []byte { // DecodeOvertakenTipRepairHold decodes a record written by EncodeOvertakenTipRepairHold. func DecodeOvertakenTipRepairHold(data []byte) ([]HeldBlock, error) { + if len(data) == 0 || data[0] != overtakenTipRepairHoldVersion { + return nil, errors.NewProcessingError("invalid overtaken-tip repair hold: not a version %d record", overtakenTipRepairHoldVersion) + } + + data = data[1:] + if len(data)%heldBlockSize != 0 { return nil, errors.NewProcessingError("invalid overtaken-tip repair hold: %d bytes is not a whole number of %d-byte entries", len(data), heldBlockSize) } diff --git a/model/overtaken_tip_repair_hold_test.go b/model/overtaken_tip_repair_hold_test.go index 0ce7979e58..900992d87d 100644 --- a/model/overtaken_tip_repair_hold_test.go +++ b/model/overtaken_tip_repair_hold_test.go @@ -18,15 +18,28 @@ func TestOvertakenTipRepairHoldRoundTrip(t *testing.T) { require.Equal(t, held, decoded) } +// No hold is still a non-empty record: over gRPC empty bytes arrive as nil, and the SQL +// state table refuses a NULL. func TestOvertakenTipRepairHoldEmpty(t *testing.T) { encoded := EncodeOvertakenTipRepairHold(nil) - require.Empty(t, encoded, "no hold is an empty record") + require.NotEmpty(t, encoded, "no hold still encodes to bytes") decoded, err := DecodeOvertakenTipRepairHold(encoded) require.NoError(t, err) require.Empty(t, decoded) } +func TestOvertakenTipRepairHoldRejectsUnknownVersion(t *testing.T) { + encoded := EncodeOvertakenTipRepairHold(nil) + encoded[0]++ + + _, err := DecodeOvertakenTipRepairHold(encoded) + require.Error(t, err) + + _, err = DecodeOvertakenTipRepairHold(nil) + require.Error(t, err, "a record without a version is not a hold this code wrote") +} + func TestOvertakenTipRepairHoldRejectsTruncated(t *testing.T) { encoded := EncodeOvertakenTipRepairHold([]HeldBlock{{Hash: chainhash.HashH([]byte("a")), ID: 7, Height: 100}}) diff --git a/services/blockvalidation/overtaken_tip_repair.go b/services/blockvalidation/overtaken_tip_repair.go index 808cc248a5..a61f0ce57a 100644 --- a/services/blockvalidation/overtaken_tip_repair.go +++ b/services/blockvalidation/overtaken_tip_repair.go @@ -559,7 +559,14 @@ func (u *BlockValidation) holdRecordedLocked(ctx context.Context, blockchainClie u.tipRepairHoldChecked = true - return len(data) > 0, nil + if len(data) == 0 { + return false, nil + } + + // One that cannot be decoded may list blocks: seed, which logs it, and write it again. + held, err := model.DecodeOvertakenTipRepairHold(data) + + return err != nil || len(held) > 0, nil } // seedHeldBlocksLocked tracks the blocks a hold recorded before the restart lists. No record diff --git a/services/blockvalidation/overtaken_tip_repair_test.go b/services/blockvalidation/overtaken_tip_repair_test.go index f029102b05..8ac5583bc7 100644 --- a/services/blockvalidation/overtaken_tip_repair_test.go +++ b/services/blockvalidation/overtaken_tip_repair_test.go @@ -20,6 +20,7 @@ import ( "github.com/bsv-blockchain/teranode/model" "github.com/bsv-blockchain/teranode/pkg/fileformat" "github.com/bsv-blockchain/teranode/services/blockchain" + "github.com/bsv-blockchain/teranode/services/blockchain/blockchain_api" "github.com/bsv-blockchain/teranode/services/subtreevalidation" "github.com/bsv-blockchain/teranode/settings" "github.com/bsv-blockchain/teranode/stores/blob" @@ -32,6 +33,7 @@ import ( "github.com/bsv-blockchain/teranode/util/test" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" ) // overtakeTestStore wraps the UTXO store so a test can act at precise points: @@ -2425,3 +2427,39 @@ func TestRepairOvertakenTips_HoldReadBackBeforeAnyOnChainWrite(t *testing.T) { require.NotZero(t, f.unminedSince(t, f.bOnly), "B is repaired") require.Empty(t, f.heldBlocks(t), "and the hold is released") } + +// wireStateClient sends SetState through a protobuf round trip, as the gRPC blockchain client +// does, before the store sees it: empty bytes arrive as nil. +type wireStateClient struct { + blockchain.ClientI +} + +func (c *wireStateClient) SetState(ctx context.Context, key string, data []byte) error { + wire, err := proto.Marshal(&blockchain_api.SetStateRequest{Key: key, Data: data}) + if err != nil { + return err + } + + var req blockchain_api.SetStateRequest + if err = proto.Unmarshal(wire, &req); err != nil { + return err + } + + return c.ClientI.SetState(ctx, req.Key, req.Data) +} + +// TestSetTxMinedStatus_EmptyHoldSurvivesTheWire: the hold with nothing in it is still written +// through the gRPC blockchain client, where empty bytes arrive as nil and the SQL state table +// refuses a NULL. A hold that cannot be written stops every block from being set mined. +func TestSetTxMinedStatus_EmptyHoldSurvivesTheWire(t *testing.T) { + f := newOvertakeFixture(t) + f.blockchainClient = &wireStateClient{ClientI: f.blockchainClient} + bv := f.newBlockValidation(t) + + require.NoError(t, bv.setTxMinedStatus(f.ctx, f.blockB.Hash())) + + _, meta, err := f.blockchainClient.GetBlockHeader(f.ctx, f.blockB.Hash()) + require.NoError(t, err) + require.True(t, meta.MinedSet) + require.Empty(t, f.heldBlocks(t)) +} From 4c65819887259f4182302ae283e647813a2def7c Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 18:10:42 +0100 Subject: [PATCH 32/33] fix(blockchain): return a not-found error for a missing state key, so its code survives gRPC From the review of the hold fix. A missing state key came back as the raw sql.ErrNoRows, which gRPC carries only as text, so block validation and the pruner told a missing hold from a failed read by matching that text. A change to it would make a fresh node's first hold read look like a failure and stall mined_set again. The SQL store now returns ERR_NOT_FOUND wrapping it; the text is kept for callers that match on it. Also: the CHANGELOG states the upgrade order (pruner no later than block validation) and what a block validation rollback leaves behind, the setTxMinedStatus comment names the 1000-block window and the give-ups, and the pruner's missing-key stub uses the real error shape. --- CHANGELOG.md | 14 ++++++++++++-- services/blockchain/server_test.go | 15 +++++++++++++++ services/blockvalidation/BlockValidation.go | 3 ++- services/pruner/worker_test.go | 2 +- stores/blockchain/sql/State.go | 8 ++++++++ stores/blockchain/sql/State_test.go | 6 +++++- 6 files changed, 43 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c77164e84d..6ba2047542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,8 +129,18 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea are repaired after it. **A single block validation instance must write mined status for the chain**: the hold is one record, so two instances can release each other's hold. A block overtaken while block validation is down and more than 10 blocks below - the tip when it starts is not repaired. No action needed for a single-instance - deployment. (#2065) + the tip when it starts is not repaired. Upgrade the pruner no later than block + validation: a newer block validation with an older pruner writes the hold but nothing + reads it. Rolling block validation back after it wrote a non-empty hold leaves the + record in place, and a newer pruner keeps holding until block validation is upgraded + again or the `OvertakenTipRepairHold` state row is deleted. No action needed for a + single-instance deployment. (#2065) +- **A missing blockchain state key is a not-found error.** `GetState` on the SQL + blockchain store returned the raw `sql.ErrNoRows` for a key never written; it now + returns an `ERR_NOT_FOUND` error wrapping it, whose code survives gRPC. Its text still + contains `sql: no rows in result set`, so callers matching on the text are unaffected; + a caller that matched `sql.ErrNoRows` with `errors.Is` must match + `errors.ErrNotFound`. (#2065) - **`GetLastNBlocks` honours `fromHeight` when orphans are included.** The SQL blockchain store ignored `fromHeight` with `includeOrphans=true` and always returned the highest blocks. It now returns only blocks at or below `fromHeight`, as documented. diff --git a/services/blockchain/server_test.go b/services/blockchain/server_test.go index aae1b9f6c8..4d5d3c47dd 100644 --- a/services/blockchain/server_test.go +++ b/services/blockchain/server_test.go @@ -2335,6 +2335,21 @@ func TestGetState(t *testing.T) { require.Contains(t, err.Error(), "forced store error") }) + t.Run("missing key is not found over gRPC", func(t *testing.T) { + storeURL, err := url.Parse("sqlitememory:///") + require.NoError(t, err) + + store, err := sql.New(logger, storeURL, tSettings) + require.NoError(t, err) + + server, err := New(ctx, logger, tSettings, store, nil) + require.NoError(t, err) + + _, err = server.GetState(ctx, &blockchain_api.GetStateRequest{Key: "never-written"}) + require.Error(t, err) + require.ErrorIs(t, errors.UnwrapGRPC(err), errors.ErrNotFound, "a client can tell a missing key from a failed read") + }) + t.Run("success - returns state", func(t *testing.T) { expectedData := []byte("some-state") diff --git a/services/blockvalidation/BlockValidation.go b/services/blockvalidation/BlockValidation.go index 912bd33c72..fe12986d5c 100644 --- a/services/blockvalidation/BlockValidation.go +++ b/services/blockvalidation/BlockValidation.go @@ -1823,7 +1823,8 @@ func (u *BlockValidation) setTxMinedStatus(ctx context.Context, blockHash *chain // The pruner processes a height only once its block has mined_set. A block the chain // has overtaken goes into the pruning hold first, so the pruner deletes nothing until - // its repair has finished, however long that takes. This reads the chain and writes the + // its repair has finished, or the repair gives the block up because its data is gone or + // it has left the 1000-block repair window. This reads the chain and writes the // hold; it repairs nothing, so a repair that keeps failing does not stop this block. Its // error is kept as text: it may be about another block, and wrapped, a block that is // gone would read as this block being gone, and the worker would not retry it. diff --git a/services/pruner/worker_test.go b/services/pruner/worker_test.go index 9950214cf0..b28297ca6d 100644 --- a/services/pruner/worker_test.go +++ b/services/pruner/worker_test.go @@ -877,7 +877,7 @@ func TestRepairHoldMissingIsNotHeld(t *testing.T) { defer cancel() client := &repairHoldClient{} - client.hold.Store(&repairHoldAnswer{err: errors.NewStorageError("no hold", sql.ErrNoRows)}) + client.hold.Store(&repairHoldAnswer{err: errors.NewNotFoundError("no state for key %s", model.OvertakenTipRepairHoldKey, sql.ErrNoRows)}) pruneSvc := &recordingPruner{calls: make(chan uint32, 2)} server := newRepairHoldTestServer(ctx, client, pruneSvc) diff --git a/stores/blockchain/sql/State.go b/stores/blockchain/sql/State.go index 0ad95e80e6..dd5fae0b0c 100644 --- a/stores/blockchain/sql/State.go +++ b/stores/blockchain/sql/State.go @@ -5,7 +5,9 @@ package sql import ( "context" + "database/sql" + "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/util/tracing" ) @@ -40,6 +42,12 @@ func (s *SQL) GetState(ctx context.Context, key string) ([]byte, error) { if err = s.db.QueryRowContext(ctx, q, key).Scan( &data, ); err != nil { + // A key never written is not found, as a code that survives gRPC; the wrapped error + // keeps sql.ErrNoRows' text for callers that match on it. + if errors.Is(err, sql.ErrNoRows) { + return nil, errors.NewNotFoundError("no state for key %s", key, err) + } + return nil, err } diff --git a/stores/blockchain/sql/State_test.go b/stores/blockchain/sql/State_test.go index 6d227bf159..f86193b1c3 100644 --- a/stores/blockchain/sql/State_test.go +++ b/stores/blockchain/sql/State_test.go @@ -6,6 +6,7 @@ import ( "net/url" "testing" + "github.com/bsv-blockchain/teranode/errors" "github.com/bsv-blockchain/teranode/ulogger" "github.com/bsv-blockchain/teranode/util/test" "github.com/stretchr/testify/require" @@ -21,8 +22,11 @@ func TestSQLGetState(t *testing.T) { s, err := New(ulogger.TestLogger{}, storeURL, tSettings) require.NoError(t, err) + // A key never written is not found, as a code a caller can match with errors.Is, + // over gRPC too; the store error's text is kept for callers that match on it. _, err = s.GetState(context.Background(), "test") - require.ErrorIs(t, err, sql.ErrNoRows) + require.ErrorIs(t, err, errors.ErrNotFound) + require.Contains(t, err.Error(), sql.ErrNoRows.Error()) err = s.SetState(context.Background(), "test", []byte("test data")) require.NoError(t, err) From 33bbf8cd5f3d585af187ea13b531150deb8395d2 Mon Sep 17 00:00:00 2001 From: Siggi Date: Thu, 29 Oct 2026 21:26:29 +0100 Subject: [PATCH 33/33] docs: correct what the CHANGELOG says about gone blocks and the not-found migration A gone block is given up after tipRepairMaxAttempts attempts, not at once like a gone subtree. Callers matching sql.ErrNoRows with errors.Is still match the new not-found error, locally and over gRPC, so nobody has to migrate; the store and server tests now pin that. The rewindblockchain comment no longer says the store returns the raw error. --- CHANGELOG.md | 14 +++++++------- .../rewindblockchain/phase3_finalize.go | 4 ++-- services/blockchain/server_test.go | 2 ++ stores/blockchain/sql/State_test.go | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ba2047542..e90df6397c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -119,9 +119,10 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea once the repair finishes. A subtree of an overtaken block that cannot be read is tried again on every block and holds pruning until it reads or the block leaves the 1000-block repair window. A subtree or block that is gone from its store can never be - read, so it is given up at once and counted in - `teranode_blockvalidation_tip_repair_abandoned_total`: its transactions may stay marked - mined while off the chain, and the pruner can then delete parents they need. `teranode_pruner_overtaken_tip_repair_held_blocks` and + read: a gone subtree is given up at once, and a gone block after 10 attempts, both + counted in `teranode_blockvalidation_tip_repair_abandoned_total`. Their transactions + may stay marked mined while off the chain, and the pruner can then delete parents they + need. `teranode_pruner_overtaken_tip_repair_held_blocks` and `teranode_blockvalidation_tip_repair_subtrees_set_aside` show a hold; a hold that stays above zero means a subtree file is corrupt or a store keeps failing, and disk use grows until it is resolved. A repair that keeps failing no longer stops healthy blocks: their @@ -137,10 +138,9 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea single-instance deployment. (#2065) - **A missing blockchain state key is a not-found error.** `GetState` on the SQL blockchain store returned the raw `sql.ErrNoRows` for a key never written; it now - returns an `ERR_NOT_FOUND` error wrapping it, whose code survives gRPC. Its text still - contains `sql: no rows in result set`, so callers matching on the text are unaffected; - a caller that matched `sql.ErrNoRows` with `errors.Is` must match - `errors.ErrNotFound`. (#2065) + returns an `ERR_NOT_FOUND` error wrapping it, whose code survives gRPC. Callers that + match `sql.ErrNoRows`, with `errors.Is` or on its text, still match, locally and over + gRPC; new callers should match `errors.ErrNotFound`. (#2065) - **`GetLastNBlocks` honours `fromHeight` when orphans are included.** The SQL blockchain store ignored `fromHeight` with `includeOrphans=true` and always returned the highest blocks. It now returns only blocks at or below `fromHeight`, as documented. diff --git a/cmd/rewindblockchain/rewindblockchain/phase3_finalize.go b/cmd/rewindblockchain/rewindblockchain/phase3_finalize.go index 2b2d84195a..656113b72b 100644 --- a/cmd/rewindblockchain/rewindblockchain/phase3_finalize.go +++ b/cmd/rewindblockchain/rewindblockchain/phase3_finalize.go @@ -59,8 +59,8 @@ func (e *env) resetBlockAssemblerState(ctx context.Context, pf *preflightResult) func (e *env) readBlockPersisterHeight(ctx context.Context) (uint32, bool, error) { existing, err := e.blockchainStore.GetState(ctx, blockPersisterHeightKey) if err != nil { - // SQL blockchain store returns sql.ErrNoRows directly for missing keys; - // future implementations may wrap into errors.ErrNotFound. + // The SQL blockchain store returns errors.ErrNotFound wrapping sql.ErrNoRows + // for a missing key; either match is enough. if errors.Is(err, sql.ErrNoRows) || errors.Is(err, errors.ErrNotFound) { e.logger.Debugf("state[%s] not present: %v", blockPersisterHeightKey, err) return 0, false, nil diff --git a/services/blockchain/server_test.go b/services/blockchain/server_test.go index 4d5d3c47dd..5e4f5a1be8 100644 --- a/services/blockchain/server_test.go +++ b/services/blockchain/server_test.go @@ -4,6 +4,7 @@ package blockchain import ( "bytes" "context" + stdsql "database/sql" "encoding/binary" "encoding/json" "fmt" @@ -2348,6 +2349,7 @@ func TestGetState(t *testing.T) { _, err = server.GetState(ctx, &blockchain_api.GetStateRequest{Key: "never-written"}) require.Error(t, err) require.ErrorIs(t, errors.UnwrapGRPC(err), errors.ErrNotFound, "a client can tell a missing key from a failed read") + require.ErrorIs(t, errors.UnwrapGRPC(err), stdsql.ErrNoRows, "callers matching the raw error still match") }) t.Run("success - returns state", func(t *testing.T) { diff --git a/stores/blockchain/sql/State_test.go b/stores/blockchain/sql/State_test.go index f86193b1c3..0d715e3e21 100644 --- a/stores/blockchain/sql/State_test.go +++ b/stores/blockchain/sql/State_test.go @@ -26,7 +26,7 @@ func TestSQLGetState(t *testing.T) { // over gRPC too; the store error's text is kept for callers that match on it. _, err = s.GetState(context.Background(), "test") require.ErrorIs(t, err, errors.ErrNotFound) - require.Contains(t, err.Error(), sql.ErrNoRows.Error()) + require.ErrorIs(t, err, sql.ErrNoRows, "callers matching the raw error still match") err = s.SetState(context.Background(), "test", []byte("test data")) require.NoError(t, err)