Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 23 additions & 17 deletions chainsource/finality.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,16 @@ var finalityBlockSubscriptionBackoffs = []time.Duration{
2 * time.Second,
}

// finalityBlockSubscriptionAttemptTimeout bounds each individual
// RegisterBlocks attempt. Without it a single hung RegisterBlocks call
// (e.g. a wedged lndclient gRPC stream) would block the conf/spend
// monitoring goroutine indefinitely — stalling Confirmed/Reorged/Done
// delivery on that watch — since the retry schedule only bounds the gaps
// between attempts, not the attempts themselves. 10s mirrors the per-call
// registration timeout used in conf_actor.go's handleRegisterConf so the
// whole file behaves consistently under a slow backend.
const finalityBlockSubscriptionAttemptTimeout = 10 * time.Second

// registerBlocksForFinality registers a block-epoch subscription used
// to synthesize a Done signal at FinalityDepth past an observed
// confirmation or spend. The call is retried with a short bounded
Expand All @@ -29,22 +39,11 @@ var finalityBlockSubscriptionBackoffs = []time.Duration{
// lndclient over gRPC); a one-shot RegisterBlocks attempt that
// briefly hiccups would leak the per-watch sub-actor indefinitely.
//
// The retries run in a dedicated arming goroutine (not the sub-actor's
// select loop), so brief blocking here is safe: more confirmation/spend
// events on this specific watch are not expected during the retry window
// (we already consumed the one that triggered the arm), and ctx
// cancellation breaks out promptly.
//
// The passed ctx MUST be the sub-actor's long-lived context, and it is
// handed to RegisterBlocks unwrapped: for in-process backends the
// block-epoch forwarder goroutine is tied to the ctx it receives, so
// bounding each attempt with a cancellable child ctx (and cancelling it
// once the call returns) would tear the subscription down the instant it
// was armed — starving finality synthesis of the very epochs it needs.
// A hung RegisterBlocks can therefore stall this arming goroutine, but
// that is contained: it is off the select loop (fix moved arming there
// precisely so a slow backend cannot wedge Confirmed/Reorged/Done
// delivery), and a genuinely wedged backend is a lost watch regardless.
// The retries run in the calling sub-actor's monitoring goroutine, so
// brief blocking here is safe: more confirmation/spend events on this
// specific watch are not expected during the retry window (we already
// consumed the one that triggered the arm), and ctx cancellation
// breaks out promptly.
//
// Returns the registration on success, or a non-nil error after
// retries are exhausted. Callers should log the error at warn level
Expand All @@ -55,7 +54,14 @@ func registerBlocksForFinality(ctx context.Context, backend ChainBackend,

var lastErr error
for attempt, backoff := range finalityBlockSubscriptionBackoffs {
reg, err := backend.RegisterBlocks(ctx)
// Bound each attempt so a hung RegisterBlocks cannot wedge the
// monitoring goroutine; the retry schedule only bounds the gaps
// between attempts, not a single stuck call.
attemptCtx, cancel := context.WithTimeout(
ctx, finalityBlockSubscriptionAttemptTimeout,
)
reg, err := backend.RegisterBlocks(attemptCtx)
cancel()
if err == nil {
return reg, nil
}
Expand Down
9 changes: 9 additions & 0 deletions darepod/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,15 @@ type UnrollConfig struct {
// MaxFeeRateSatPerVByte caps fee estimates to prevent runaway
// fees. Zero uses the default of 100 sat/vB.
MaxFeeRateSatPerVByte int64 `mapstructure:"maxfeeratesatpervbyte"`

// ReconcileProbeTimeoutSec bounds each per-anchor restart-
// reconciliation probe issued by the chainsource-backed
// ChainReconciler (in seconds). A probe that times out is
// treated as "not on chain" and triggers a conservative
// rollback of the affected anchor; operators running against
// a slow backend can raise this to avoid spurious rollbacks.
// Zero uses the reconciler's internal default (10s).
ReconcileProbeTimeoutSec int64 `mapstructure:"reconcileprobetimeoutsec"`
}

// FeeEstimationConfig groups optional external chain fee providers used by the
Expand Down
47 changes: 47 additions & 0 deletions darepod/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -5299,6 +5299,36 @@ func (s *Server) initUnrollSubsystem(ctx context.Context,
exitObserver = fn.Some[actor.TellOnlyRef[vtxo.ManagerMsg]](ref)
})

// Build a chainsource-backed reconciler factory so every per-
// target actor verifies its checkpoint anchors against the
// canonical chain on restart. Without this, a daemon that was
// offline during a reorg would silently resume against stale
// PlannerState — see unroll/reconcile.go for the safety
// rationale.
//
// The factory bakes the target outpoint into the chainsource
// caller-ID prefix so two actors that reconcile the same
// shared proof-graph ancestor concurrently land on distinct
// service keys (chainsource keys on (CallerID, Txid, PkScript,
// TargetConfs)); a static prefix would collide.
reconcileLog := s.subLogger("UREC")
probeTimeout := s.unrollReconcileProbeTimeout()
reconcilerFactory := func(target wire.OutPoint,
proof *recovery.Proof) unroll.ChainReconciler {

return unroll.NewChainSourceReconciler(
unroll.ChainSourceReconcilerConfig{
ChainSource: chainSourceRef,
Proof: proof,
CallerID: fmt.Sprintf(
"unroll-reconcile-%s", target,
),
ProbeTimeout: probeTimeout,
Log: fn.Some(reconcileLog),
},
)
}

registry := unroll.NewUnrollRegistryActor(unroll.RegistryConfig{
Store: &unroll.DBRegistryStore{
UEStore: ueStore,
Expand All @@ -5319,6 +5349,9 @@ func (s *Server) initUnrollSubsystem(ctx context.Context,
Preimage: preimages,
},
VTXOExitObserver: exitObserver,
ChainReconcilerFactory: fn.Some(
unroll.ChainReconcilerFactory(reconcilerFactory),
),
})
s.unrollRegistry = registry
s.unrollRegistryRef = fn.Some(registry.Ref())
Expand Down Expand Up @@ -5659,6 +5692,20 @@ func (s *Server) unrollMaxFeeRate() int64 {
return 0
}

// unrollReconcileProbeTimeout returns the configured per-anchor probe
// timeout for the chainsource-backed restart reconciler, or zero so
// the reconciler falls back to defaultReconcileProbeTimeout.
func (s *Server) unrollReconcileProbeTimeout() time.Duration {
if s.cfg.Unroll != nil &&
s.cfg.Unroll.ReconcileProbeTimeoutSec > 0 {
return time.Duration(
s.cfg.Unroll.ReconcileProbeTimeoutSec,
) * time.Second
}

return 0
}

// unrollBumpAfterBlocks returns the configured fee-bump cadence (in
// blocks) for the shared txconfirm actor used by the unroll subsystem,
// or zero to let txconfirm fall back to DefaultFeeBumpIntervalBlocks.
Expand Down
8 changes: 8 additions & 0 deletions sample-darepod.conf
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,14 @@
# Maximum unroll fee rate in sat/vB. The zero value uses the unroll default.
# unroll.maxfeeratesatpervbyte=0

# Per-anchor restart-reconciliation probe timeout in seconds. The chainsource-
# backed ChainReconciler issues one probe per persisted anchor on Resume; a
# probe that exceeds this budget is treated as "not on chain" and triggers a
# conservative rollback of the affected anchor. Operators running against a
# slow chain backend can raise this to avoid spurious rollbacks. The zero
# value uses the reconciler's internal default (10s).
# unroll.reconcileprobetimeoutsec=0

# Swap server gRPC address for swapruntime builds.
# swap.serveraddress=localhost:10030

Expand Down
Loading
Loading