fix(subtreevalidation): validate peer subtrees skipped during catchup once RUNNING again

CHANGELOG.md

@@ -724,6 +724,20 @@ Upgrading from v0.16.0 changes nothing for them; upgrading from an earlier relea
 
 ### Not breaking, but worth knowing
 
+- **Peer subtrees announced during a catchup are validated afterwards instead of
+  lost.** While catching up, the subtree validator used to drop every
+  peer-announced subtree, so the next block arrived with all of them unvalidated
+  and had to read their transactions from the UTXO store. Up to
+  `subtreevalidation_skippedSubtreeBackfillMax` (default 4096, newest kept) of them
+  are now validated through the normal peer-subtree path once the node is RUNNING
+  again, using at most half of `subtreevalidation_peerSubtreeConcurrency`, so
+  expect a burst of peer subtree validation right after a catchup. Subtrees
+  announced while IDLE, or more than 30 minutes before their replay, are still
+  dropped. New metric:
+  `teranode_subtreevalidation_skipped_peer_subtrees_total{outcome}` (recorded,
+  evicted, expired, requeued, dropped, replayed, replay_failed). Set the key to `0` for the previous
+  behaviour. (#2055)
+
 - **Mining candidates no longer build on the old tip during a reorg.** While block
   assembly is in the `reorging`, `reconciling` or `resetting` state, it can still
   hold the tip from before the reorg. `GetMiningCandidate` built candidates on that

docs/references/settings/services/subtreevalidation_settings.md

@@ -15,6 +15,7 @@
 | ProcessTxMetaUsingCacheConcurrency | int | 32 | subtreevalidation_processTxMetaUsingCache_Concurrency | **CRITICAL** - Cache-probe workers shared by all subtrees being validated (server-wide, not per subtree; peer subtrees and block validation each get a pool of this size) |
 | PeerSubtreeConcurrency | int | 32 | subtreevalidation_peerSubtreeConcurrency | Peer-announced subtrees validated at once (each holds ~1 GiB per 1M txs); taken before the quorum lock, so block validation never waits behind the queue. Block subtrees do not use it. 0 or less disables the limit |
 | TxMetaTombstoneWindow | time.Duration | 30m | subtreevalidation_txMetaTombstoneWindow | How long a txmeta DELETE (a tx demoted to conflicting) keeps that tx out of the txmeta cache, against late ADDs and read-through writes; entries live one to two windows, about 60 MB per million demoted txs. Must exceed the worst txmeta consumer lag; must be positive |
+| SkippedSubtreeBackfillMax | int | 4096 | subtreevalidation_skippedSubtreeBackfillMax | Peer subtrees announced while catching up that are kept and validated once the node is RUNNING again (deduplicated by hash, oldest dropped when full, replayed with at most half the peer-subtree slots). 0 or less turns the backfill off |
 | ProcessTxMetaUsingCacheMissingTxThreshold | int | 1 | subtreevalidation_processTxMetaUsingCache_MissingTxThreshold | Cache miss threshold |
 | BlockHeightRetentionAdjustment | int32 | 0 | subtreevalidation_blockHeightRetentionAdjustment | Adjustment applied to `GlobalBlockHeightRetention` (see `GetSubtreeValidationBlockHeightRetention`) |
 | SubtreeDAHConcurrency | int | 8 | subtreevalidation_subtreeDAHConcurrency | DAH processing concurrency |

services/subtreevalidation/Server.go

@@ -6,6 +6,7 @@ package subtreevalidation
 import (
 	"context"
 	"net/http"
+	"net/url"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -165,6 +166,18 @@ type Server struct {
 	// peerSubtreeAdmission bounds how many peer subtrees are validated at
 	// once; see subtreesHandler.
 	peerSubtreeAdmission admissionGate
+
+	// skippedSubtrees keeps peer subtrees announced while the node was catching up, for
+	// validation once it is RUNNING again.
+	skippedSubtrees skippedSubtrees
+
+	// replaySubtreeFn replaces the validation of a replayed subtree, after its admission, age and
+	// RUNNING checks; nil in production.
+	replaySubtreeFn func(ctx context.Context, hash *chainhash.Hash, baseURL *url.URL, peerID string) error
+
+	// watcherReplayCalledFn is called by a RUNNING watcher after its replay call returns; nil in
+	// production.
+	watcherReplayCalledFn func()
 }
 
 // New creates a new Server instance with the provided dependencies.
@@ -424,6 +437,9 @@ func (u *Server) blockchainSubscriptionListener(ctx context.Context) {
 							}
 							ctxLogger.Errorf("[SubtreeValidation:blockchainSubscriptionListener] failed to update best block: %s", err)
 						}
+
+						// A block can arrive before any peer subtree message after a catchup.
+						u.replaySkippedSubtreesIfRunning(ctx)
 					}
 				}
 			}

services/subtreevalidation/metrics.go

@@ -146,6 +146,13 @@ var (
 	// a metric a spike in the rate would be invisible.
 	prometheusSubtreeAlreadyExistsSkipped prometheus.Counter
 
+	// prometheusSkippedSubtrees counts peer subtrees announced while the node was catching up, by
+	// outcome: recorded (newly kept), evicted (oldest dropped when full), expired (announced more
+	// than skippedSubtreeMaxAge before its replay), requeued (kept again because the node was
+	// catching up again), dropped (the node was IDLE at replay), replayed (validated or already
+	// stored), replay_failed.
+	prometheusSkippedSubtrees *prometheus.CounterVec
+
 	// prometheusPeerSubtreeAdmissionWait tracks how long a peer subtree waited for a
 	// subtreevalidation_peerSubtreeConcurrency slot, and prometheusPeerSubtreeAdmissionWaiting
 	// how many are waiting now. Together they show whether the limit is too low.
@@ -373,6 +380,16 @@ func _initPrometheusMetrics() {
 		},
 	)
 
+	prometheusSkippedSubtrees = promauto.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "teranode",
+			Subsystem: "subtreevalidation",
+			Name:      "skipped_peer_subtrees_total",
+			Help:      "Peer subtrees announced while the node was catching up, by outcome (recorded, evicted, expired, requeued, dropped, replayed, replay_failed)",
+		},
+		[]string{"outcome"},
+	)
+
 	prometheusPeerSubtreeAdmissionWait = promauto.NewHistogram(
 		prometheus.HistogramOpts{
 			Namespace: "teranode",

services/subtreevalidation/peer_subtree_admission_test.go

@@ -217,6 +217,8 @@ func TestSubtreesHandler_SkipsWhenNoLongerRunningAfterTheWait(t *testing.T) {
 	catchingBlocks := blockchain.FSMStateCATCHINGBLOCKS
 	blockchainClient := &blockchain.Mock{}
 	blockchainClient.On("GetFSMCurrentState", mock.Anything).Return(&catchingBlocks, nil)
+	// The dropped subtree is kept for the backfill, which waits for RUNNING; let it give up.
+	blockchainClient.On("WaitForFSMtoTransitionToGivenState", mock.Anything, blockchain.FSMStateRUNNING).Return(errors.NewContextCanceledError("test")).Maybe()
 	server.blockchainClient = blockchainClient
 
 	hash := chainhash.HashH([]byte("peer subtree"))