From c903b8fd795d27dd5b731579331ff721d0dea694 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:16:57 -0700 Subject: [PATCH 1/9] routing: report attempt outcomes back to the payment session In this commit, we let a payment session hear what became of the routes it handed out. Today the payment lifecycle reports every attempt to mission control and to nothing else, which is enough while the only view of network liquidity in the system is the node wide one mission control owns. A session that carries a view of its own has no way to see the results of its own work, so we give it one. The new PaymentResultReporter interface is optional. The lifecycle type asserts on it after it has finished talking to mission control, and a session that does not implement it never hears a thing, which is exactly what the session lnd ships expects. Nothing about the existing flow moves: mission control still decides whether a failure is terminal, and the reporting happens after that decision rather than instead of it. The third method is the one worth explaining. A session that tracks the attempts it produced needs to know when no further outcome is coming, because a route it returned may never have reached the switch at all, in which case nothing else would ever tell it so. The lifecycle calls it on the way out, from a defer beside the one that stops the shard goroutines. We also lift the two additional edge helpers off the session and onto the edge map itself, so that a second session implementation can embed the map and pick up both interface methods without copying them. --- routing/payment_lifecycle.go | 41 ++++++++++ routing/payment_session.go | 142 ++++++++++++++++++++++------------- 2 files changed, 131 insertions(+), 52 deletions(-) diff --git a/routing/payment_lifecycle.go b/routing/payment_lifecycle.go index 2b8180c23d7..b242a7cb425 100644 --- a/routing/payment_lifecycle.go +++ b/routing/payment_lifecycle.go @@ -206,6 +206,14 @@ func (p *paymentLifecycle) resumePayment(ctx context.Context) ([32]byte, // return. defer p.stop() + // This session will never be asked for another route, and no further + // outcome will be reported to it, so let it drop whatever it was + // tracking on behalf of the attempts it handed out. A route we asked + // for but never managed to send is only ever cleaned up here. + defer p.reportToSession(func(r PaymentResultReporter) { + r.ReleaseAttempts() + }) + // If we had any existing attempts outstanding, we'll start by spinning // up goroutines that'll collect their results and deliver them to the // lifecycle loop below. @@ -466,6 +474,25 @@ func (p *paymentLifecycle) requestRoute(ctx context.Context, return nil, nil } +// reportToSession hands an attempt outcome to the payment session when the +// session asked to hear about them. Sessions that do not implement +// PaymentResultReporter, the stock session among them, are left alone. +// +// NOTE: p.paySession can be nil when this is reached through SendToRoute, +// where there is no payment lifecycle driving the attempts. +func (p *paymentLifecycle) reportToSession(report func(PaymentResultReporter)) { + if p.paySession == nil { + return + } + + reporter, ok := p.paySession.(PaymentResultReporter) + if !ok { + return + } + + report(reporter) +} + // stop signals any active shard goroutine to exit. func (p *paymentLifecycle) stop() { close(p.quit) @@ -876,6 +903,14 @@ func (p *paymentLifecycle) handleSwitchErr(ctx context.Context, reason = &internalErrorReason } + // If the payment session keeps a belief state of its own, it + // needs the same observation mission control just received. + p.reportToSession(func(r PaymentResultReporter) { + r.ReportAttemptFailure( + attemptID, &attempt.Route, srcIdx, msg, + ) + }) + // Fail the attempt only if there's no reason. if reason == nil { // Fail the attempt. @@ -1206,6 +1241,12 @@ func (p *paymentLifecycle) handleAttemptResult(ctx context.Context, log.Errorf("Error reporting payment success to mc: %v", err) } + // If the payment session keeps a belief state of its own, it needs the + // same observation mission control just received. + p.reportToSession(func(r PaymentResultReporter) { + r.ReportAttemptSuccess(attempt.AttemptID, &attempt.Route) + }) + // In case of success we atomically store settle result to the DB and // move the shard to the settled state. htlcAttempt, err := p.router.cfg.Control.SettleAttempt( diff --git a/routing/payment_session.go b/routing/payment_session.go index 4cddfa2eaca..d6018b19f38 100644 --- a/routing/payment_session.go +++ b/routing/payment_session.go @@ -158,6 +158,95 @@ type PaymentSession interface { channelID uint64) *models.CachedEdgePolicy } +// PaymentResultReporter is an optional interface that a PaymentSession may +// implement when it keeps a view of the network that it needs to update from +// the outcome of the attempts it handed out. The payment lifecycle reports +// every attempt outcome to mission control, which is a node global store the +// session cannot own; a session with its own belief state needs the same +// stream of observations delivered to itself as well. +// +// A session that does not implement this interface simply never hears about +// the attempts it produced, which is exactly the contract the stock session +// was written against. +type PaymentResultReporter interface { + // ReportAttemptSuccess informs the session that the given attempt + // settled. The route is the one the session returned from + // RequestRoute. + ReportAttemptSuccess(attemptID uint64, rt *route.Route) + + // ReportAttemptFailure informs the session that the given attempt + // failed. The failure source index follows the convention used by + // mission control: a nil index means the failure could not be + // attributed to any node on the route, and a zero index means the + // failure happened at our own node. The failure message is nil when it + // could not be read. + ReportAttemptFailure(attemptID uint64, rt *route.Route, + failureSourceIdx *int, failure lnwire.FailureMessage) + + // ReleaseAttempts tells the session that its lifecycle has finished and + // that no further outcome will be reported to it. A session that tracks + // the attempts it handed out needs this, because a route it returned + // may never have reached the switch at all, in which case nothing else + // would ever tell it so. + ReleaseAttempts() +} + +// additionalEdges is the set of ephemeral edges that a payment session knows +// about on top of the channel graph, keyed by the node the edge starts at. +// These come from bolt 11 route hints or from the blinded paths of an offer. +type additionalEdges map[route.Vertex][]AdditionalEdge + +// UpdateAdditionalEdge updates the channel edge policy for a private edge. It +// validates the message signature and checks it's up to date, then applies the +// updates to the supplied policy. It returns a boolean to indicate whether +// there's an error when applying the updates. +func (a additionalEdges) UpdateAdditionalEdge(msg *lnwire.ChannelUpdate1, + pubKey *btcec.PublicKey, policy *models.CachedEdgePolicy) bool { + + // Validate the message signature. + if err := netann.VerifyChannelUpdateSignature(msg, pubKey); err != nil { + log.Errorf( + "Unable to validate channel update signature: %v", err, + ) + return false + } + + // Update channel policy for the additional edge. + policy.TimeLockDelta = msg.TimeLockDelta + policy.FeeBaseMSat = lnwire.MilliSatoshi(msg.BaseFee) + policy.FeeProportionalMillionths = lnwire.MilliSatoshi(msg.FeeRate) + + log.Debugf("New private channel update applied: %v", + lnutils.SpewLogClosure(msg)) + + return true +} + +// GetAdditionalEdgePolicy uses the public key and channel ID to query the +// ephemeral channel edge policy for additional edges. Returns a nil if nothing +// found. +func (a additionalEdges) GetAdditionalEdgePolicy(pubKey *btcec.PublicKey, + channelID uint64) *models.CachedEdgePolicy { + + target := route.NewVertex(pubKey) + + edges, ok := a[target] + if !ok { + return nil + } + + for _, edge := range edges { + policy := edge.EdgePolicy() + if policy.ChannelID != channelID { + continue + } + + return policy + } + + return nil +} + // paymentSession is used during an HTLC routings session to prune the local // chain view in response to failures, and also report those failures back to // MissionController. The snapshot copied for this session will only ever grow, @@ -169,7 +258,7 @@ type PaymentSession interface { type paymentSession struct { selfNode route.Vertex - additionalEdges map[route.Vertex][]AdditionalEdge + additionalEdges getBandwidthHints func(Graph) (bandwidthHints, error) @@ -457,54 +546,3 @@ func (p *paymentSession) RequestRoute(maxAmt, feeLimit lnwire.MilliSatoshi, return route, err } } - -// UpdateAdditionalEdge updates the channel edge policy for a private edge. It -// validates the message signature and checks it's up to date, then applies the -// updates to the supplied policy. It returns a boolean to indicate whether -// there's an error when applying the updates. -func (p *paymentSession) UpdateAdditionalEdge(msg *lnwire.ChannelUpdate1, - pubKey *btcec.PublicKey, policy *models.CachedEdgePolicy) bool { - - // Validate the message signature. - if err := netann.VerifyChannelUpdateSignature(msg, pubKey); err != nil { - log.Errorf( - "Unable to validate channel update signature: %v", err, - ) - return false - } - - // Update channel policy for the additional edge. - policy.TimeLockDelta = msg.TimeLockDelta - policy.FeeBaseMSat = lnwire.MilliSatoshi(msg.BaseFee) - policy.FeeProportionalMillionths = lnwire.MilliSatoshi(msg.FeeRate) - - log.Debugf("New private channel update applied: %v", - lnutils.SpewLogClosure(msg)) - - return true -} - -// GetAdditionalEdgePolicy uses the public key and channel ID to query the -// ephemeral channel edge policy for additional edges. Returns a nil if nothing -// found. -func (p *paymentSession) GetAdditionalEdgePolicy(pubKey *btcec.PublicKey, - channelID uint64) *models.CachedEdgePolicy { - - target := route.NewVertex(pubKey) - - edges, ok := p.additionalEdges[target] - if !ok { - return nil - } - - for _, edge := range edges { - policy := edge.EdgePolicy() - if policy.ChannelID != channelID { - continue - } - - return policy - } - - return nil -} From 212fceefc24d7e6f73e7627028007a9ec20f3e19 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:17:23 -0700 Subject: [PATCH 2/9] routing: add the liquidity interval belief model and store In this commit, we add the thing the router reasons over. Mission control remembers a penalty per node pair and lets it fade on a half life. This remembers an amount range per directed channel: the largest amount it has watched pass, the smallest it has watched fail, and an estimate in between, with a confidence and a classification saying which side of the bimodal liquidity distribution the channel appears to sit on. Three kinds of observation write to it. A probe is what we learn when a failure comes back from a node further along the route than a given hop, which proves that hop forwarded. A failure drops the upper bound. A settlement is different in kind from both: the balance really has moved, so the forward interval slides down by the settled amount and the reverse interval slides up by the same. Every observation writes both directions, because liquidity sitting on one side of a funding output is not sitting on the other, and nothing in mission control makes that inference. The probability model on top is a branch table ordered by how much the evidence proves, from a bound we watched hold down to a bimodal prior we have nothing behind. The prior expresses its scale as a fraction of capacity rather than as an absolute number of millisatoshis, which is what lets it mean the same thing on channels that differ in size by orders of magnitude. There is no clock anywhere in the file. Two properties are worth calling out because they are load bearing rather than incidental. A belief seeded in from durable storage is marked as restored, and the model then refuses to let either of its bounds speak with certainty: this model has no clock, so an amount it calls impossible is never attempted, and the attempt is the only thing that could correct a bound that has gone stale. And the store carries an overlay of the amounts we currently have committed in flight, kept apart from the beliefs and never persisted, because it records what we are doing to the network rather than what we believe about it. --- routing/interval_belief.go | 831 +++++++++++++++++++++++++++++++++ routing/interval_store.go | 575 +++++++++++++++++++++++ routing/interval_store_test.go | 615 ++++++++++++++++++++++++ 3 files changed, 2021 insertions(+) create mode 100644 routing/interval_belief.go create mode 100644 routing/interval_store.go create mode 100644 routing/interval_store_test.go diff --git a/routing/interval_belief.go b/routing/interval_belief.go new file mode 100644 index 00000000000..ca252acb13f --- /dev/null +++ b/routing/interval_belief.go @@ -0,0 +1,831 @@ +package routing + +import ( + "math" + + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +// The constants below parameterize the liquidity interval belief model. They +// were selected by an evolutionary search against a payment simulator rather +// than derived from first principles, so each one is named and documented here +// for what it does rather than for why that particular number is right. Where a +// constant expresses a fraction of a channel's capacity it is written as a +// fraction rather than an absolute amount, which is what lets the model carry +// across networks whose channels differ in size by orders of magnitude. +const ( + // intervalPriorFloor is the probability floor of the bimodal prior. It + // keeps an amount in the middle of a channel's range from being ruled + // out entirely when we have no evidence at all. + intervalPriorFloor = 0.005 + + // intervalPriorMass is the probability mass assigned to each of the two + // modes of the prior, the depleted one near zero and the saturated one + // near capacity. + intervalPriorMass = 0.495 + + // intervalPriorScale is the width of both modes of the prior, as a + // fraction of the channel's capacity. + intervalPriorScale = 0.018 + + // intervalPriorCliff is the point, as a fraction of capacity, at which + // the saturated mode falls off. Above it a channel is assumed unable to + // forward even if it has never failed. + intervalPriorCliff = 0.965 + + // intervalPriorMax and intervalPriorMin clamp the prior away from + // certainty in either direction. + intervalPriorMax = 0.999 + intervalPriorMin = 0.0005 + + // intervalProvenProbability is the probability assigned to an amount at + // or below an amount this channel has already forwarded. It is not one, + // because liquidity can move underneath us between attempts. + intervalProvenProbability = 0.9985 + + // intervalLocalProbability is the probability assigned to one of our + // own channels that the bandwidth hints say can carry the amount. The + // small haircut below one is what makes the search prefer shorter + // routes without a separate hop count term. + intervalLocalProbability = 0.9995 + + // intervalRichBase, intervalRichConfidence and intervalRichMargin + // combine into the probability of a channel we have classified as + // saturated and whose estimate covers the amount. + intervalRichBase = 0.975 + intervalRichConfidence = 0.022 + intervalRichMargin = 0.002 + + // intervalEstimateBase, intervalEstimateConfidence and + // intervalEstimateMargin combine into the probability of an + // unclassified channel whose estimate covers the amount. + intervalEstimateBase = 0.90 + intervalEstimateConfidence = 0.075 + intervalEstimateMargin = 0.02 + + // intervalPositionFloor, intervalPositionMass and + // intervalPositionExponent shape the interpolation across a known + // interval. At the lower bound the probability is near certainty, at + // the upper bound it is near zero, and the exponent controls how fast + // it falls off in between. + intervalPositionFloor = 0.01 + intervalPositionMass = 0.94 + intervalPositionExponent = 2.8 + + // intervalPriorBlend is the weight the prior keeps when an interval is + // known on both sides. Holding a little prior back stops a single pair + // of observations from speaking with more authority than it has. + intervalPriorBlend = 0.10 + + // intervalOverBase and intervalOverScale price an amount that runs past + // what we estimate the channel holds, without a hard upper bound to + // rule it out. + intervalOverBase = 0.12 + intervalOverScale = 0.035 + + // intervalLowModeFloor and intervalLowModeMass shape the exponential + // tail used for a channel classified as depleted. + intervalLowModeFloor = 0.006 + intervalLowModeMass = 0.78 + + // intervalLowModeProven is the probability of an amount at or below the + // proven lower bound of a depleted channel. + intervalLowModeProven = 0.998 + + // intervalLowModeEstimate is the probability floor applied to a + // depleted channel whose estimate still covers the amount. + intervalLowModeEstimate = 0.82 + + // intervalUnknownCapacity is the probability used for a channel whose + // capacity we do not know, which happens for light clients and for hop + // hints. Without a capacity none of the fractions above mean anything, + // so we fall back to a flat guess. + intervalUnknownCapacity = 0.6 + + // intervalMaxProbability and intervalMinProbability clamp the output of + // the model. The minimum is deliberately not zero: only a proven upper + // bound is allowed to say impossible. + intervalMaxProbability = 0.999 + intervalMinProbability = 0.000001 + + // intervalRestoredFloor and intervalRestoredCeiling clamp the output of + // the model for a belief that was restored from disk rather than + // gathered in this process. Neither certainty is available to a belief + // that has been asleep: the floor keeps a stale upper bound from ruling + // an amount out for good, and the ceiling keeps a stale lower bound from + // being trusted as if we had just watched it hold. + intervalRestoredFloor = 0.012 + intervalRestoredCeiling = 0.95 + + // intervalRestoredConfidence is the factor applied to the confidence of + // a belief when it is restored, since whatever evidence stood behind it + // is now at least one restart old. + intervalRestoredConfidence = 0.5 +) + +// Liquidity mode classifications. The model does not just carry a probability +// curve, it commits to a hypothesis about which side of the bimodal +// distribution a channel sits on and then reasons inside that hypothesis. +const ( + // intervalModeDepleted means the channel appears to be nearly empty in + // this direction. + intervalModeDepleted int8 = -1 + + // intervalModeUnknown means we have not classified the channel. + intervalModeUnknown int8 = 0 + + // intervalModeRich means the channel appears to hold nearly its whole + // capacity in this direction. + intervalModeRich int8 = 1 +) + +// Thresholds that gate the classification and the estimate updates. These are +// written as integer divisors of capacity so that they can be applied without +// leaving millisatoshi arithmetic. +const ( + // intervalStrongObservationDivisor sets how large an observation has to + // be, relative to capacity, before it is allowed to move the mode + // latch. A dust sized probe proves almost nothing about which mode a + // channel is in, so it moves the bounds but not the classification. + intervalStrongObservationDivisor = 200 + + // intervalDepletedDivisor is the fraction of capacity below which an + // estimate classifies the channel as depleted. + intervalDepletedDivisor = 50 + + // intervalRichNumerator and intervalRichDenominator give the fraction + // of capacity above which an estimate classifies the channel as rich. + intervalRichNumerator = 49 + intervalRichDenominator = 50 + + // intervalProbeEstimateNumerator and intervalProbeEstimateDenominator + // give the estimate we jump to when a strong observation proves a + // channel forwarded an amount: we assume it is near the top of its + // range rather than exactly at the amount we saw. + intervalProbeEstimateNumerator = 97 + intervalProbeEstimateDenominator = 100 + + // intervalFailureEstimateDivisor is the divisor applied to a failing + // amount to get the collapsed estimate after a failure. + intervalFailureEstimateDivisor = 32 + + // intervalFailureFloorDivisor caps the collapsed estimate of a strongly + // observed failure at this fraction of capacity. + intervalFailureFloorDivisor = 1000 +) + +// Confidence levels latched by each kind of observation. Confidence here is a +// saturating latch on how much evidence we have seen rather than a posterior +// width, and it only ever feeds the probability model as a small additive term. +const ( + intervalProbeConfidence = 0.94 + intervalProbeReverseConfidence = 0.86 + intervalFailureConfidence = 0.99 + intervalFailureReverseConfidence = 0.97 + intervalSettleConfidence = 0.96 +) + +// intervalRetryLadder prices the question "this channel just refused X, what do +// I believe about a smaller amount?". Each rung is a ratio of the new amount to +// the failed amount, paired with the factor the probability is multiplied by. +// Retrying at three quarters of a failed amount is nearly hopeless; retrying at +// a hundredth of it is nearly fine. This ladder is what replaces blacklisting a +// channel and waiting for a penalty to decay. +var intervalRetryLadder = []struct { + ratio float64 + factor float64 +}{ + {ratio: 0.75, factor: 0.004}, + {ratio: 0.40, factor: 0.018}, + {ratio: 0.15, factor: 0.075}, + {ratio: 0.04, factor: 0.30}, + {ratio: 0.01, factor: 0.62}, +} + +// intervalRetryFloor is the factor used below the last rung of the ladder. +const intervalRetryFloor = 0.88 + +// IntervalKey identifies one direction of one channel. Unlike mission control, +// which keys its history on a node pair, the interval model keys on the +// directed channel, because the quantity it tracks is the balance sitting on +// one side of one funding output. +// +// NOTE: under non-strict forwarding a node may forward over a sibling channel +// to the same peer, in which case an observation can land on the wrong key. The +// pair keyed model does not have that problem, and this is the price the +// directed key pays for being able to hold an amount interval that means +// something physical. +type IntervalKey struct { + // ChanID is the short channel id of the channel. + ChanID uint64 + + // From is the node the liquidity is flowing away from. + From route.Vertex + + // To is the node the liquidity is flowing towards. + To route.Vertex +} + +// intervalPairScopeChanID marks a belief held about a node pair as a whole +// rather than about one channel between them. Zero is safe to use for this +// because it is not a short channel id any real channel can have: it would name +// the first output of the first transaction of the genesis block. +const intervalPairScopeChanID = 0 + +// Reverse returns the key for the opposite direction of the same channel. +func (k IntervalKey) Reverse() IntervalKey { + return IntervalKey{ + ChanID: k.ChanID, + From: k.To, + To: k.From, + } +} + +// PairScope returns the key describing the node pair this channel connects, +// which is the granularity an observation has to fall back to when it cannot +// name a channel. +func (k IntervalKey) PairScope() IntervalKey { + return IntervalKey{ + ChanID: intervalPairScopeChanID, + From: k.From, + To: k.To, + } +} + +// IsPairScoped reports whether this key describes a node pair rather than a +// channel. +func (k IntervalKey) IsPairScoped() bool { + return k.ChanID == intervalPairScopeChanID +} + +// intervalScopeKey returns the key an observation about a hop should be written +// under, given how many channels connect the pair the hop crosses. +// +// A channel is the granularity this model wants, because the quantity it tracks +// is the balance sitting on one side of one funding output. It is not always +// the granularity the evidence supports. Under non-strict forwarding a node +// asked to forward over one channel may use any channel it has to the same +// peer, and an onion failure names neither. So when a pair has more than one +// channel, an observation is written about the pair instead. +// +// The alternative, writing the same observation onto every channel of the pair, +// is worse rather than merely coarser. This model's upper bound is hard: an +// amount at or above it is impossible, and no amount of reduced confidence +// softens that, because confidence enters the model as a small additive term +// and never as a multiplier on the bound. Spreading a failure across siblings +// would therefore assert something false and unrecoverable about every channel +// that was not the one to refuse. Pair scope asserts only what was observed, +// which is that this peer could not move this amount to that node. It is also +// the granularity mission control has always used, so it is a loss of +// resolution rather than a loss of correctness. +func intervalScopeKey(key IntervalKey, siblings int) IntervalKey { + if siblings > 1 { + return key.PairScope() + } + + return key +} + +// LiquidityInterval is what we believe about the liquidity available in one +// direction of one channel. It is an interval rather than a point estimate: +// LowerOK is the largest amount we have proven can pass, UpperFail the smallest +// amount we have proven cannot, and Estimate our best guess in between. +// +// There is deliberately no clock in this structure. A bound moves when new +// evidence arrives or when a settlement displaces liquidity, and never merely +// because time has passed. +type LiquidityInterval struct { + // LowerOK is the largest amount this direction has been proven to + // carry. Anything at or below it is treated as near certain. + LowerOK lnwire.MilliSatoshi + + // UpperFail is the smallest amount this direction has been proven not + // to carry. Zero means no failure has been observed. Anything at or + // above it is treated as impossible. + UpperFail lnwire.MilliSatoshi + + // Estimate is our best guess at the balance currently available. + Estimate lnwire.MilliSatoshi + + // Confidence is a saturating measure of how much evidence stands behind + // the estimate, in the range [0, 1]. + Confidence float64 + + // Failures and Successes count the observations that have landed here. + Failures uint32 + Successes uint32 + + // Mode is the classification latch, one of the intervalMode constants. + Mode int8 + + // Known is set once any observation has been recorded. + Known bool + + // Restored marks a belief that came back from disk rather than from an + // attempt this process made. It is cleared by the first fresh + // observation, because from that point the bounds describe evidence we + // gathered ourselves. + Restored bool +} + +// markRestored turns a belief loaded from disk into soft evidence. The bounds +// are kept, since they are still the best guess anybody has about a channel we +// have not touched yet, but the confidence behind them is cut and the +// probability model is told to stop short of certainty in either direction. +func (l *LiquidityInterval) markRestored() { + l.Restored = true + l.Confidence *= intervalRestoredConfidence +} + +// normalize restores the invariant 0 <= LowerOK <= Estimate < UpperFail <= +// capacity and re-runs the mode classification. It is applied on every read and +// every write, so that no caller ever sees an interval that contradicts itself. +// When an upper bound contradicts a lower bound it is the upper bound that is +// dropped, because the lower bound records something we watched succeed. +func (l *LiquidityInterval) normalize(capacity lnwire.MilliSatoshi) { + if l.LowerOK > capacity { + l.LowerOK = capacity + } + + if l.UpperFail > capacity { + l.UpperFail = 0 + } + if l.UpperFail != 0 && l.LowerOK >= l.UpperFail { + l.UpperFail = 0 + } + + if l.Estimate < l.LowerOK { + l.Estimate = l.LowerOK + } + if l.Estimate > capacity { + l.Estimate = capacity + } + if l.UpperFail != 0 && l.Estimate >= l.UpperFail { + l.Estimate = l.UpperFail - 1 + if l.Estimate < l.LowerOK { + l.Estimate = l.LowerOK + } + } + + if capacity == 0 { + return + } + + switch { + case l.Estimate <= capacity/intervalDepletedDivisor: + l.Mode = intervalModeDepleted + + case l.Estimate >= capacity*intervalRichNumerator/ + intervalRichDenominator: + + l.Mode = intervalModeRich + } +} + +// intervalStrongObservation reports whether an observation of the given amount +// is large enough to be allowed to move the mode latch. +func intervalStrongObservation(amt, capacity lnwire.MilliSatoshi) bool { + if capacity == 0 { + return false + } + + threshold := capacity / intervalStrongObservationDivisor + if threshold < 1 { + threshold = 1 + } + + return amt >= threshold +} + +// intervalPrior returns the success probability of forwarding the given amount +// over a channel of the given capacity, in the absence of any observation. It +// is the bimodal hypothesis written directly as a probability: channels are +// assumed to sit near one end of their range or the other, so a small amount is +// near certain to pass and an amount close to the whole capacity is near +// certain to fail, with a narrow transition between the two. +// +// Both the width of the modes and the position of the cliff are fractions of +// capacity, which is what makes this prior mean the same thing on a channel of +// any size. +func intervalPrior(amt, capacity lnwire.MilliSatoshi) float64 { + if capacity == 0 || amt == 0 || amt > capacity { + return 0 + } + + ratio := float64(amt) / float64(capacity) + + lowSide := intervalPriorMass * math.Exp(-ratio/intervalPriorScale) + highSide := intervalPriorMass / + (1 + math.Exp((ratio-intervalPriorCliff)/intervalPriorScale)) + + probability := intervalPriorFloor + lowSide + highSide + + return math.Min( + math.Max(probability, intervalPriorMin), intervalPriorMax, + ) +} + +// intervalRetryFactor returns the multiplier to apply to the probability of an +// amount when this payment has already watched the same channel refuse a larger +// amount. An amount at or above the failed one is hopeless; below it the ladder +// gives back belief in proportion to how much smaller the retry is. +func intervalRetryFactor(amt, failedAt lnwire.MilliSatoshi) float64 { + if failedAt == 0 { + return 1 + } + if amt >= failedAt { + return 0 + } + + ratio := float64(amt) / float64(failedAt) + for _, rung := range intervalRetryLadder { + if ratio > rung.ratio { + return rung.factor + } + } + + return intervalRetryFloor +} + +// lowModeProbability returns the probability of an amount over a channel we +// have classified as depleted. The available balance is modelled as an +// exponential tail rising from the proven lower bound, truncated and +// renormalized at the proven upper bound when we have one. +func (l *LiquidityInterval) lowModeProbability(amt, + capacity lnwire.MilliSatoshi) float64 { + + if l.LowerOK >= amt { + return intervalLowModeProven + } + if l.UpperFail != 0 && amt >= l.UpperFail { + return 0 + } + + scale := math.Max(float64(capacity)*intervalPriorScale, 1) + tail := math.Exp(-float64(amt-l.LowerOK) / scale) + probability := intervalLowModeFloor + intervalLowModeMass*tail + + if l.Estimate >= amt { + probability = math.Max(probability, intervalLowModeEstimate) + } + + // If we know where the channel stops, the tail cannot run past that + // point, so we cut it there and renormalize what is left. + if l.UpperFail != 0 { + upperTail := math.Exp( + -float64(l.UpperFail-l.LowerOK) / scale, + ) + if upperTail < 0.999 { + tail = math.Max((tail-upperTail)/(1-upperTail), 0) + probability = intervalLowModeFloor + + intervalLowModeMass*tail + } + } + + return probability +} + +// Probability returns the success probability of forwarding the given amount +// over the channel this interval describes. +func (l *LiquidityInterval) Probability(amt, + capacity lnwire.MilliSatoshi) float64 { + + // Without a capacity none of the fractions in the model mean anything, + // so fall back to a flat guess rather than pretending to know. + if capacity == 0 { + return intervalUnknownCapacity + } + + // An amount larger than the channel itself is impossible whatever we + // remember about it, so this one zero is never softened below. + prior := intervalPrior(amt, capacity) + if prior == 0 { + return 0 + } + + probability := l.rawProbability(amt, capacity, prior) + + // A belief we restored from disk describes a network that has had every + // chance to move on since we wrote it down. The bounds are still worth + // something, which is why we keep them, but they are no longer allowed + // to speak with certainty in either direction: a restored upper bound + // says unlikely rather than impossible, and a restored lower bound says + // likely rather than proven. Without the floor the model has no way back + // from a bound that has gone stale, because nothing but an attempt can + // revise one and an impossible amount is never attempted. + if l.Restored { + return math.Min( + math.Max(probability, intervalRestoredFloor), + intervalRestoredCeiling, + ) + } + + // A bound this process watched hold is the one thing the model is + // allowed to call impossible, so it is not floored. + if probability == 0 { + return 0 + } + + return math.Min( + math.Max(probability, intervalMinProbability), + intervalMaxProbability, + ) +} + +// rawProbability runs the branch table of the model. The branches are ordered +// by how much the evidence proves, from a bound we watched hold to a guess we +// have nothing behind. A zero here means the evidence rules the amount out; it +// is the caller that decides whether the evidence is fresh enough to be +// believed that far. +func (l *LiquidityInterval) rawProbability(amt, capacity lnwire.MilliSatoshi, + prior float64) float64 { + + var probability float64 + + switch { + // We have watched this channel carry at least this much. + case l.LowerOK >= amt: + probability = intervalProvenProbability + + // We have watched this channel refuse at most this much. + case l.UpperFail != 0 && amt >= l.UpperFail: + return 0 + + // Nothing has ever been observed here. + case !l.Known: + probability = prior + + // The channel looks empty in this direction. + case l.Mode == intervalModeDepleted: + probability = l.lowModeProbability(amt, capacity) + + // The channel looks full in this direction and our estimate covers the + // amount. + case l.Mode == intervalModeRich && l.Estimate >= amt: + margin := float64(l.Estimate-amt+1) / float64(capacity) + probability = intervalRichBase + + intervalRichConfidence*l.Confidence + + intervalRichMargin*math.Min(margin*8, 1) + + // We have bounds on both sides, so interpolate across them. The + // position of the amount inside the interval is what decides, which + // makes this a smooth walk between the two certainties above. + case l.UpperFail != 0: + lower := float64(l.LowerOK) + upper := float64(l.UpperFail) + position := (float64(amt) - lower) / + math.Max(upper-lower, 1) + position = math.Min(math.Max(position, 0), 1) + + probability = intervalPositionFloor + intervalPositionMass* + math.Pow(1-position, intervalPositionExponent) + probability = (1-intervalPriorBlend)*probability + + intervalPriorBlend*prior + + // No upper bound, but our estimate covers the amount. + case l.Estimate >= amt: + margin := float64(l.Estimate-amt+1) / float64(capacity) + probability = intervalEstimateBase + + intervalEstimateConfidence*l.Confidence + + intervalEstimateMargin*math.Min(margin*5, 1) + + // The amount runs past our estimate but nothing has proven it cannot + // pass, so discount the prior by how far past it runs. + default: + over := float64(amt-l.Estimate) / float64(capacity) + probability = prior * intervalOverBase * + math.Exp(-over/intervalOverScale) + } + + return probability +} + +// recordProbe records that this direction forwarded the given amount, which we +// learn whenever a failure comes back from a node further along the route than +// this hop. The forward lower bound rises to the amount, and the reverse +// direction's upper bound drops, because liquidity sitting on this side of the +// channel cannot also be sitting on the other side. +func (l *LiquidityInterval) recordProbe(reverse *LiquidityInterval, + amt, capacity lnwire.MilliSatoshi) { + + if amt > l.LowerOK { + l.LowerOK = amt + } + if l.UpperFail != 0 && amt >= l.UpperFail { + l.UpperFail = 0 + } + + inferred := amt + strong := intervalStrongObservation(amt, capacity) + if strong { + high := capacity * intervalProbeEstimateNumerator / + intervalProbeEstimateDenominator + if high > inferred { + inferred = high + } + + l.Mode = intervalModeRich + } + + if !l.Known || inferred > l.Estimate { + l.Estimate = inferred + } + + l.Known = true + l.Restored = false + l.Confidence = math.Max(l.Confidence, intervalProbeConfidence) + l.Successes++ + if l.Failures > 0 { + l.Failures-- + } + l.normalize(capacity) + + // Whatever sits on this side of the channel is not on the other side, + // so an amount that passed here bounds what can pass back. + reverseUpper := lnwire.MilliSatoshi(1) + if capacity >= amt { + reverseUpper = capacity - amt + 1 + } + if reverse.UpperFail == 0 || reverseUpper < reverse.UpperFail { + reverse.UpperFail = reverseUpper + } + if reverse.LowerOK >= reverse.UpperFail { + reverse.LowerOK = reverse.UpperFail - 1 + } + + if strong { + reverseEstimate := capacity - l.Estimate + if reverseEstimate < reverse.LowerOK { + reverseEstimate = reverse.LowerOK + } + if !reverse.Known || reverseEstimate < reverse.Estimate { + reverse.Estimate = reverseEstimate + } + + reverse.Mode = intervalModeDepleted + } + + reverse.Known = true + reverse.Restored = false + reverse.Confidence = math.Max( + reverse.Confidence, intervalProbeReverseConfidence, + ) + reverse.normalize(capacity) +} + +// recordFailure records that this direction could not carry the given amount. +// The forward upper bound drops to the failing amount, and the reverse +// direction gains a lower bound and counts a success, because a failure in one +// direction is evidence of available liquidity in the other. +func (l *LiquidityInterval) recordFailure(reverse *LiquidityInterval, + amt, capacity lnwire.MilliSatoshi) { + + if l.UpperFail == 0 || amt < l.UpperFail { + l.UpperFail = amt + } + if l.LowerOK >= amt { + l.LowerOK = amt - 1 + } + + strong := intervalStrongObservation(amt, capacity) + depleted := amt / intervalFailureEstimateDivisor + if strong { + floor := capacity / intervalFailureFloorDivisor + if floor < 1 { + floor = 1 + } + if depleted > floor { + depleted = floor + } + + l.Mode = intervalModeDepleted + } + + if depleted < l.LowerOK { + depleted = l.LowerOK + } + if !l.Known || depleted < l.Estimate { + l.Estimate = depleted + } + + l.Known = true + l.Restored = false + l.Confidence = math.Max(l.Confidence, intervalFailureConfidence) + l.Failures++ + l.normalize(capacity) + + reverseLower := lnwire.MilliSatoshi(0) + if capacity >= amt { + reverseLower = capacity - amt + 1 + } + if reverseLower > reverse.LowerOK { + reverse.LowerOK = reverseLower + } + if reverse.UpperFail != 0 && reverse.LowerOK >= reverse.UpperFail { + reverse.UpperFail = 0 + } + + reverseEstimate := capacity - l.Estimate + if reverseEstimate < reverse.LowerOK { + reverseEstimate = reverse.LowerOK + } + if !reverse.Known || reverseEstimate > reverse.Estimate { + reverse.Estimate = reverseEstimate + } + if strong { + reverse.Mode = intervalModeRich + } + + reverse.Known = true + reverse.Restored = false + reverse.Confidence = math.Max( + reverse.Confidence, intervalFailureReverseConfidence, + ) + reverse.Successes++ + reverse.normalize(capacity) +} + +// recordSettlement records that this direction actually moved the given amount. +// Unlike the other two observations this one does not just narrow an interval, +// it shifts it: the balance really has moved across the channel, so the forward +// interval slides down by the settled amount and the reverse interval slides up +// by the same. +func (l *LiquidityInterval) recordSettlement(reverse *LiquidityInterval, + amt, capacity lnwire.MilliSatoshi) { + + // Work out what the balance must have been before the settlement, then + // subtract what just left. + before := l.Estimate + if !l.Known || before < amt { + before = amt + if intervalStrongObservation(amt, capacity) { + high := capacity * intervalProbeEstimateNumerator / + intervalProbeEstimateDenominator + if high > before { + before = high + } + + l.Mode = intervalModeRich + } + } + if before > capacity { + before = capacity + } + + l.Estimate = before - amt + if l.LowerOK > amt { + l.LowerOK -= amt + } else { + l.LowerOK = 0 + } + if l.UpperFail > amt { + l.UpperFail -= amt + } else { + l.UpperFail = 0 + } + + l.Known = true + l.Restored = false + l.Confidence = math.Max(l.Confidence, intervalSettleConfidence) + l.Successes++ + if l.Failures > 0 { + l.Failures-- + } + l.normalize(capacity) + + headroom := lnwire.MilliSatoshi(0) + if capacity >= amt { + headroom = capacity - amt + } + + if reverse.LowerOK > headroom { + reverse.LowerOK = capacity + } else { + reverse.LowerOK += amt + } + + if reverse.UpperFail != 0 { + if reverse.UpperFail > headroom { + reverse.UpperFail = 0 + } else { + reverse.UpperFail += amt + } + } + + reverse.Estimate = capacity - l.Estimate + if reverse.Estimate < reverse.LowerOK { + reverse.Estimate = reverse.LowerOK + } + + reverse.Known = true + reverse.Restored = false + reverse.Confidence = math.Max( + reverse.Confidence, intervalSettleConfidence, + ) + reverse.Successes++ + if reverse.Failures > 0 { + reverse.Failures-- + } + reverse.normalize(capacity) +} diff --git a/routing/interval_store.go b/routing/interval_store.go new file mode 100644 index 00000000000..2c82ee175e3 --- /dev/null +++ b/routing/interval_store.go @@ -0,0 +1,575 @@ +package routing + +import ( + "context" + "fmt" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/lightningnetwork/lnd/lnwire" +) + +// DefaultMaxIntervalHistory is the default number of directed channels the +// interval store will remember. Entries are created in pairs, one per +// direction, so this is roughly half as many channels. Mission control bounds +// its own history the same way, for the same reason: a long lived node would +// otherwise accumulate an entry for every channel it has ever touched. +const DefaultMaxIntervalHistory = 10000 + +// DefaultIntervalFlushInterval is how often the store writes accumulated +// beliefs down when a persister is attached. +const DefaultIntervalFlushInterval = time.Second + +// intervalEvictionFraction is the fraction of the store dropped when it grows +// past its bound. Evicting a batch rather than a single entry keeps the cost of +// eviction amortized rather than paid on every insert once the store is full. +const intervalEvictionFraction = 4 + +// intervalEntry is one directed channel's belief plus the bookkeeping the store +// needs to bound itself. +type intervalEntry struct { + LiquidityInterval + + // seq is the value of the store's counter when this entry was last + // written, which is what eviction orders on. + seq uint64 +} + +// PersistedInterval is one belief as it is written to and read from durable +// storage. +type PersistedInterval struct { + // Key identifies the directed channel the belief is about. + Key IntervalKey + + // Interval is the belief itself. + Interval LiquidityInterval +} + +// IntervalPersister is the durable backing of an IntervalStore. It is an +// interface rather than a concrete store so that the routing package does not +// have to care which database is underneath, and so that a node with no SQL +// backend configured simply runs without one. +type IntervalPersister interface { + // FetchIntervals returns at most limit of the most recently written + // beliefs. + FetchIntervals(ctx context.Context, limit int) ([]PersistedInterval, + error) + + // StoreIntervals writes the given beliefs, replacing any already held + // for the same directed channels. + StoreIntervals(ctx context.Context, intervals []PersistedInterval) error + + // PruneIntervals drops all but the given number of most recently + // written beliefs. + PruneIntervals(ctx context.Context, keep int) error + + // PurgeIntervals drops every stored belief. + PurgeIntervals(ctx context.Context) error +} + +// IntervalStore holds the router's belief about the liquidity of every directed +// channel it has observed. It plays the role mission control plays for the +// stock router, with two differences that matter. It records amount intervals +// rather than penalties, and it never forgets anything on a timer: a bound +// moves when new evidence arrives, not when a half life elapses. +// +// The store lives for as long as the node does and is shared by every payment, +// which is what makes the beliefs one payment gathers available to the next. +// With a persister attached it also outlives the process, in which case the +// beliefs it reads back are marked restored, since a bound written down before +// a restart describes a network that has had every chance to move on. +type IntervalStore struct { + started atomic.Bool + stopped atomic.Bool + + mu sync.Mutex + + // entries holds one belief per directed channel. + entries map[IntervalKey]*intervalEntry + + // maxEntries bounds the size of the store. + maxEntries int + + // seq is a monotonic counter used to order entries for eviction. + seq uint64 + + // persister is the durable backing, or nil when the store is memory + // only. Everything below it is unused in that case. + persister IntervalPersister + + // flushInterval is how often accumulated changes are written down. + flushInterval time.Duration + + // dirty holds the keys written since the last flush. + dirty map[IntervalKey]struct{} + + // held tracks the amounts this node currently has committed on + // directed channels through HTLCs it has sent and not yet seen + // resolved. It is summed across every payment in flight, so that one + // payment prices a corridor knowing what another payment is already + // holding on it. + // + // This is not part of what the store believes about the network and is + // never persisted. It records what we are doing to the network right + // now, which is knowledge that expires the moment the HTLC does. + held map[IntervalKey]lnwire.MilliSatoshi + + quit chan struct{} + wg sync.WaitGroup +} + +// NewIntervalStore builds an empty store bounded at the given number of +// directed channels. A non-positive bound selects the default. The store is +// memory only until a persister is attached. +func NewIntervalStore(maxEntries int) *IntervalStore { + if maxEntries <= 0 { + maxEntries = DefaultMaxIntervalHistory + } + + return &IntervalStore{ + entries: make(map[IntervalKey]*intervalEntry), + maxEntries: maxEntries, + held: make(map[IntervalKey]lnwire.MilliSatoshi), + quit: make(chan struct{}), + } +} + +// UsePersistence attaches durable storage to the store. It must be called +// before Start. +func (s *IntervalStore) UsePersistence(persister IntervalPersister, + flushInterval time.Duration) { + + s.mu.Lock() + defer s.mu.Unlock() + + if flushInterval <= 0 { + flushInterval = DefaultIntervalFlushInterval + } + + s.persister = persister + s.flushInterval = flushInterval + s.dirty = make(map[IntervalKey]struct{}) +} + +// Start loads whatever beliefs were written down before this process began and +// starts the goroutine that writes new ones. It is a no-op on a store with no +// persister, which is what a node running without a SQL backend has. +func (s *IntervalStore) Start(ctx context.Context) error { + if !s.started.CompareAndSwap(false, true) { + return nil + } + + s.mu.Lock() + persister := s.persister + limit := s.maxEntries + s.mu.Unlock() + + if persister == nil { + return nil + } + + stored, err := persister.FetchIntervals(ctx, limit) + if err != nil { + return fmt.Errorf("unable to load liquidity intervals: %w", err) + } + + for _, entry := range stored { + s.Restore(entry.Key, entry.Interval) + } + + // Reading beliefs in is not a reason to write them straight back out. + s.mu.Lock() + clear(s.dirty) + s.mu.Unlock() + + log.Infof("Loaded %d liquidity interval beliefs", len(stored)) + + s.wg.Add(1) + go s.flusher() + + return nil +} + +// Stop writes down anything still pending and stops the flush goroutine. +func (s *IntervalStore) Stop() error { + if !s.started.Load() || !s.stopped.CompareAndSwap(false, true) { + return nil + } + + close(s.quit) + s.wg.Wait() + + // A shutdown is the one moment we know there will be no further + // observations, so it is worth paying for a last write. + return s.flush(context.Background()) +} + +// flusher writes accumulated changes down on a ticker. +// +// NOTE: this must be run as a goroutine. +func (s *IntervalStore) flusher() { + defer s.wg.Done() + + s.mu.Lock() + interval := s.flushInterval + s.mu.Unlock() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + if err := s.flush(context.Background()); err != nil { + log.Errorf("Unable to flush liquidity "+ + "intervals: %v", err) + } + + case <-s.quit: + return + } + } +} + +// flush writes every belief changed since the last call. +// +// A ticker rather than a write on every observation is deliberate. One payment +// attempt writes both directions of every hop it touched, so a write through +// store would put a handful of database round trips on the path between an +// HTLC failing and the next route being chosen, which is the one path in the +// router that a user waits on. Nothing here needs to survive a crash to stay +// correct either: a belief that never reached disk is a belief the router +// rediscovers on its next attempt, at the cost of that attempt. Mission control +// batches its own writes for the same reasons. +func (s *IntervalStore) flush(ctx context.Context) error { + s.mu.Lock() + + if s.persister == nil || len(s.dirty) == 0 { + s.mu.Unlock() + + return nil + } + + pending := make([]PersistedInterval, 0, len(s.dirty)) + for key := range s.dirty { + entry, ok := s.entries[key] + if !ok { + continue + } + + pending = append(pending, PersistedInterval{ + Key: key, + Interval: entry.LiquidityInterval, + }) + } + + // Clear the dirty set before releasing the lock. An observation that + // lands during the write below marks its key again, so the worst case + // is that we write it twice rather than lose it. + clear(s.dirty) + + persister := s.persister + limit := s.maxEntries + s.mu.Unlock() + + if err := persister.StoreIntervals(ctx, pending); err != nil { + return err + } + + return persister.PruneIntervals(ctx, limit) +} + +// Get returns the belief held for the given directed channel, normalized +// against the given capacity. A channel that has never been observed returns +// the zero interval, which the probability model reads as "no evidence". +func (s *IntervalStore) Get(key IntervalKey, + capacity lnwire.MilliSatoshi) LiquidityInterval { + + s.mu.Lock() + defer s.mu.Unlock() + + entry, ok := s.entries[key] + if !ok { + return LiquidityInterval{} + } + + interval := entry.LiquidityInterval + interval.normalize(capacity) + + return interval +} + +// Probability returns the success probability of forwarding the given amount +// over the given directed channel. +func (s *IntervalStore) Probability(key IntervalKey, + amt, capacity lnwire.MilliSatoshi) float64 { + + interval := s.Get(key, capacity) + + return interval.Probability(amt, capacity) +} + +// RecordProbe records that the given directed channel forwarded the given +// amount, which we learn whenever a failure is reported by a node further along +// the route than this hop. +func (s *IntervalStore) RecordProbe(key IntervalKey, + amt, capacity lnwire.MilliSatoshi) { + + s.update(key, amt, capacity, func(forward, reverse *LiquidityInterval, + amt lnwire.MilliSatoshi) { + + forward.recordProbe(reverse, amt, capacity) + }) +} + +// RecordFailure records that the given directed channel could not carry the +// given amount. +func (s *IntervalStore) RecordFailure(key IntervalKey, + amt, capacity lnwire.MilliSatoshi) { + + s.update(key, amt, capacity, func(forward, reverse *LiquidityInterval, + amt lnwire.MilliSatoshi) { + + forward.recordFailure(reverse, amt, capacity) + }) +} + +// RecordSettlement records that the given directed channel actually moved the +// given amount, which shifts both directions of the interval rather than merely +// narrowing them. +func (s *IntervalStore) RecordSettlement(key IntervalKey, + amt, capacity lnwire.MilliSatoshi) { + + s.update(key, amt, capacity, func(forward, reverse *LiquidityInterval, + amt lnwire.MilliSatoshi) { + + forward.recordSettlement(reverse, amt, capacity) + }) +} + +// update applies an observation to both directions of a channel under the +// store's lock. Observations of a zero amount, or of a channel whose capacity +// we do not know, carry no information the model can use and are dropped. The +// sanitized amount is handed to the callback, which must use it in place of the +// amount its caller was given. +func (s *IntervalStore) update(key IntervalKey, amt, + capacity lnwire.MilliSatoshi, + apply func(forward, reverse *LiquidityInterval, + amt lnwire.MilliSatoshi)) { + + if amt == 0 || capacity == 0 { + return + } + + // An amount larger than the capacity cannot be a real observation about + // this channel. It can still reach us, because the capacity we path + // find against is a synthetic one when a peer has several channels to + // the same node, so clamp rather than reject. + if amt > capacity { + amt = capacity + } + + s.mu.Lock() + defer s.mu.Unlock() + + forward := s.entryLocked(key) + reverse := s.entryLocked(key.Reverse()) + + apply(&forward.LiquidityInterval, &reverse.LiquidityInterval, amt) + + // Every observation writes both directions, so both need writing down. + s.markDirtyLocked(key) + s.markDirtyLocked(key.Reverse()) + + s.evictLocked() +} + +// entryLocked returns the entry for a key, creating it if needed, and stamps it +// as the most recently written. +// +// NOTE: the store's mutex must be held. +func (s *IntervalStore) entryLocked(key IntervalKey) *intervalEntry { + entry, ok := s.entries[key] + if !ok { + entry = &intervalEntry{} + s.entries[key] = entry + } + + s.seq++ + entry.seq = s.seq + + return entry +} + +// markDirtyLocked records that a key needs writing down. It is a no-op on a +// store with no persister, which is what keeps the memory only path free of +// any bookkeeping it would never read. +// +// NOTE: the store's mutex must be held. +func (s *IntervalStore) markDirtyLocked(key IntervalKey) { + if s.dirty == nil { + return + } + + s.dirty[key] = struct{}{} +} + +// evictLocked drops the least recently written entries when the store has grown +// past its bound. +// +// NOTE: the store's mutex must be held. +func (s *IntervalStore) evictLocked() { + if len(s.entries) <= s.maxEntries { + return + } + + keys := make([]IntervalKey, 0, len(s.entries)) + for key := range s.entries { + keys = append(keys, key) + } + + sort.Slice(keys, func(i, j int) bool { + return s.entries[keys[i]].seq < s.entries[keys[j]].seq + }) + + drop := len(s.entries) / intervalEvictionFraction + for _, key := range keys[:drop] { + delete(s.entries, key) + + if s.dirty != nil { + delete(s.dirty, key) + } + } +} + +// Restore seeds the store with a belief that was held before this process +// started. The interval is taken as it was written down, but it is marked as +// restored, which stops the probability model from treating either of its +// bounds as a certainty until a fresh observation replaces it. +// +// An entry that has already been observed in this process is left alone, since +// what we have watched ourselves beats anything we read back. +func (s *IntervalStore) Restore(key IntervalKey, + interval LiquidityInterval) { + + s.mu.Lock() + defer s.mu.Unlock() + + if existing, ok := s.entries[key]; ok && existing.Known && + !existing.Restored { + + return + } + + entry := s.entryLocked(key) + entry.LiquidityInterval = interval + entry.Known = true + entry.markRestored() + + s.evictLocked() + + // A restored belief is already on disk, and the halved confidence it + // now carries is a reading of it rather than a new observation, so + // there is nothing here worth writing back. +} + +// ForEach hands every belief the store holds to the callback, which is how a +// persistence layer reads out what needs writing down. +func (s *IntervalStore) ForEach(cb func(IntervalKey, LiquidityInterval)) { + s.mu.Lock() + defer s.mu.Unlock() + + for key, entry := range s.entries { + cb(key, entry.LiquidityInterval) + } +} + +// Held returns the amount this node currently has committed on the given +// directed channel through HTLCs it has sent and not yet seen resolved. +func (s *IntervalStore) Held(key IntervalKey) lnwire.MilliSatoshi { + s.mu.Lock() + defer s.mu.Unlock() + + return s.held[key] +} + +// Hold records that we have committed the given amounts, one per directed +// channel of a route we are about to send over. +func (s *IntervalStore) Hold(amounts map[IntervalKey]lnwire.MilliSatoshi) { + s.mu.Lock() + defer s.mu.Unlock() + + for key, amt := range amounts { + s.held[key] += amt + } +} + +// Release gives back amounts recorded by an earlier call to Hold, which the +// caller makes once the HTLC that committed them has resolved. +// +// An amount larger than what is held would mean the caller has released +// something twice. That must not happen, but if it does we would rather forget +// a hold than carry a phantom one, because a hold nothing is behind depresses a +// channel for every payment and nothing but another release can lift it. +func (s *IntervalStore) Release(amounts map[IntervalKey]lnwire.MilliSatoshi) { + s.mu.Lock() + defer s.mu.Unlock() + + for key, amt := range amounts { + current, ok := s.held[key] + if !ok { + continue + } + + if current <= amt { + delete(s.held, key) + + continue + } + + s.held[key] = current - amt + } +} + +// HeldLen returns the number of directed channels currently carrying a hold. +// It exists so that a test can assert that nothing leaked. +func (s *IntervalStore) HeldLen() int { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.held) +} + +// Clear forgets everything the store has learned. It exists so that an operator +// can reset the router's beliefs the way mission control's history can be +// reset. +func (s *IntervalStore) Clear(ctx context.Context) error { + s.mu.Lock() + + s.entries = make(map[IntervalKey]*intervalEntry) + s.held = make(map[IntervalKey]lnwire.MilliSatoshi) + s.seq = 0 + + persister := s.persister + if s.dirty != nil { + clear(s.dirty) + } + s.mu.Unlock() + + if persister == nil { + return nil + } + + return persister.PurgeIntervals(ctx) +} + +// Len returns the number of directed channels the store currently holds a +// belief for. +func (s *IntervalStore) Len() int { + s.mu.Lock() + defer s.mu.Unlock() + + return len(s.entries) +} diff --git a/routing/interval_store_test.go b/routing/interval_store_test.go new file mode 100644 index 00000000000..087613004a8 --- /dev/null +++ b/routing/interval_store_test.go @@ -0,0 +1,615 @@ +package routing + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +var ( + // testIntervalCapacity is the capacity used by the interval tests. It + // is large enough that the fractional thresholds of the model land on + // distinct amounts. + testIntervalCapacity = lnwire.MilliSatoshi(1_000_000_000) + + // testIntervalKey is the directed channel the tests observe. + testIntervalKey = IntervalKey{ + ChanID: 7, + From: route.Vertex{1}, + To: route.Vertex{2}, + } +) + +// TestIntervalNormalize tests that the interval invariants hold no matter what +// combination of bounds is written into the structure. +func TestIntervalNormalize(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + + tests := []struct { + name string + interval LiquidityInterval + check func(*testing.T, LiquidityInterval) + }{ + { + name: "upper bound contradicting lower bound is " + + "dropped", + interval: LiquidityInterval{ + LowerOK: 500, + UpperFail: 400, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.EqualValues(t, 500, l.LowerOK) + require.EqualValues(t, 0, l.UpperFail) + }, + }, + { + name: "bounds above capacity are clamped", + interval: LiquidityInterval{ + LowerOK: capacity + 1, + UpperFail: capacity + 5, + Estimate: capacity + 9, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.Equal(t, capacity, l.LowerOK) + require.EqualValues(t, 0, l.UpperFail) + require.Equal(t, capacity, l.Estimate) + }, + }, + { + name: "estimate is pulled inside the interval", + interval: LiquidityInterval{ + LowerOK: 100, + UpperFail: 1000, + Estimate: 5000, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.EqualValues(t, 999, l.Estimate) + }, + }, + { + name: "estimate is raised to the lower bound", + interval: LiquidityInterval{ + LowerOK: 100, + Estimate: 10, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.EqualValues(t, 100, l.Estimate) + }, + }, + { + name: "a low estimate classifies as depleted", + interval: LiquidityInterval{ + Estimate: capacity / 100, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.Equal( + t, intervalModeDepleted, l.Mode, + ) + }, + }, + { + name: "a high estimate classifies as rich", + interval: LiquidityInterval{ + Estimate: capacity, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.Equal(t, intervalModeRich, l.Mode) + }, + }, + { + name: "a middling estimate stays unclassified", + interval: LiquidityInterval{ + Estimate: capacity / 2, + }, + check: func(t *testing.T, l LiquidityInterval) { + require.Equal(t, intervalModeUnknown, l.Mode) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + interval := test.interval + interval.normalize(capacity) + + // The invariant must hold in every case. + require.LessOrEqual(t, interval.LowerOK, + interval.Estimate) + require.LessOrEqual(t, interval.Estimate, capacity) + if interval.UpperFail != 0 { + require.Less(t, interval.Estimate, + interval.UpperFail) + require.LessOrEqual(t, interval.UpperFail, + capacity) + } + + test.check(t, interval) + }) + } +} + +// TestIntervalPrior tests the shape of the bimodal prior: near certainty for +// tiny amounts, a cliff near capacity, and scale invariance across channel +// sizes. +func TestIntervalPrior(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + + // A dust amount is nearly certain to pass. + require.Greater(t, intervalPrior(capacity/10000, capacity), 0.95) + + // An amount taking the whole channel is unlikely to pass. The cliff of + // the saturated mode sits just under capacity rather than at it, so + // what is left here is the tail of that mode, not zero. + require.Less(t, intervalPrior(capacity, capacity), 0.1) + + // The prior is monotonically decreasing in the amount. + previous := 1.0 + for i := 1; i <= 100; i++ { + amt := capacity / 100 * lnwire.MilliSatoshi(i) + current := intervalPrior(amt, capacity) + require.LessOrEqual(t, current, previous) + previous = current + } + + // The prior depends on the ratio, not the absolute amount, so the same + // fraction of a channel a thousand times larger prices the same. + big := capacity * 1000 + for _, fraction := range []lnwire.MilliSatoshi{2, 5, 10, 100} { + require.InDelta( + t, intervalPrior(capacity/fraction, capacity), + intervalPrior(big/fraction, big), 1e-9, + ) + } + + // An amount larger than the capacity is impossible. + require.Zero(t, intervalPrior(capacity+1, capacity)) +} + +// TestIntervalRetryFactor tests that the retry ladder gives back belief in +// proportion to how much smaller a retry is than the amount that failed. +func TestIntervalRetryFactor(t *testing.T) { + t.Parallel() + + failedAt := lnwire.MilliSatoshi(1000) + + // Without a failure the factor leaves the probability alone. + require.EqualValues(t, 1, intervalRetryFactor(500, 0)) + + // At or above the failed amount there is no point in trying. + require.Zero(t, intervalRetryFactor(1000, failedAt)) + require.Zero(t, intervalRetryFactor(1500, failedAt)) + + // Below it, the factor rises as the retry shrinks. + previous := 0.0 + for _, amt := range []lnwire.MilliSatoshi{ + 900, 500, 300, 100, 20, 5, + } { + current := intervalRetryFactor(amt, failedAt) + require.Greater(t, current, previous) + previous = current + } + + // A retry at a thousandth of the failed amount is nearly unaffected. + require.EqualValues( + t, intervalRetryFloor, intervalRetryFactor(1, failedAt), + ) +} + +// TestIntervalProbabilityBranches tests that the probability model orders its +// answers by how much the evidence proves. +func TestIntervalProbabilityBranches(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + // With no capacity the model has no scale to work with and falls back + // to a flat guess. + var empty LiquidityInterval + require.EqualValues( + t, intervalUnknownCapacity, empty.Probability(amt, 0), + ) + + // With no observations at all the model returns the prior. + require.Equal( + t, intervalPrior(amt, capacity), + empty.Probability(amt, capacity), + ) + + // A proven lower bound is near certainty. + proven := LiquidityInterval{Known: true, LowerOK: amt} + require.EqualValues( + t, intervalProvenProbability, + proven.Probability(amt, capacity), + ) + + // A proven upper bound is exactly zero, which is the only way the model + // says impossible. + failed := LiquidityInterval{Known: true, UpperFail: amt} + require.Zero(t, failed.Probability(amt, capacity)) + require.Zero(t, failed.Probability(amt+1, capacity)) + + // Inside a known interval the probability falls off with position, from + // near the proven bound to near the failed one. + interval := LiquidityInterval{ + Known: true, + LowerOK: capacity / 10, + UpperFail: capacity / 2, + } + interval.normalize(capacity) + + low := interval.Probability(capacity/10+1, capacity) + mid := interval.Probability(capacity/4, capacity) + high := interval.Probability(capacity/2-1, capacity) + + require.Greater(t, low, mid) + require.Greater(t, mid, high) + require.Greater(t, low, 0.5) + require.Less(t, high, 0.2) + + // Every branch stays inside the clamps. + for _, l := range []LiquidityInterval{ + empty, proven, failed, interval, + } { + for i := 1; i <= 100; i++ { + p := l.Probability( + capacity/100*lnwire.MilliSatoshi(i), capacity, + ) + require.GreaterOrEqual(t, p, 0.0) + require.LessOrEqual(t, p, intervalMaxProbability) + } + } +} + +// TestIntervalStoreProbe tests that a forwarded amount raises the forward lower +// bound and bounds the reverse direction, because liquidity on one side of a +// channel is not on the other. +func TestIntervalStoreProbe(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + capacity := testIntervalCapacity + amt := capacity / 4 + + store.RecordProbe(testIntervalKey, amt, capacity) + + forward := store.Get(testIntervalKey, capacity) + require.True(t, forward.Known) + require.Equal(t, amt, forward.LowerOK) + require.EqualValues(t, 0, forward.UpperFail) + require.Equal(t, intervalModeRich, forward.Mode) + require.EqualValues(t, 1, forward.Successes) + + // The amount we just watched pass is now near certain. + require.EqualValues( + t, intervalProvenProbability, + store.Probability(testIntervalKey, amt, capacity), + ) + + // The reverse direction cannot hold what the forward direction just + // proved it holds. + reverse := store.Get(testIntervalKey.Reverse(), capacity) + require.True(t, reverse.Known) + require.Equal(t, capacity-amt+1, reverse.UpperFail) + require.Equal(t, intervalModeDepleted, reverse.Mode) + require.Zero(t, store.Probability( + testIntervalKey.Reverse(), capacity-amt+1, capacity, + )) + + // A larger probe raises the bound further. + store.RecordProbe(testIntervalKey, amt*2, capacity) + forward = store.Get(testIntervalKey, capacity) + require.Equal(t, amt*2, forward.LowerOK) + + // A smaller one does not lower it. + store.RecordProbe(testIntervalKey, amt, capacity) + forward = store.Get(testIntervalKey, capacity) + require.Equal(t, amt*2, forward.LowerOK) +} + +// TestIntervalStoreFailure tests that a failure drops the forward upper bound +// and is read as evidence of liquidity in the reverse direction. +func TestIntervalStoreFailure(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + capacity := testIntervalCapacity + amt := capacity / 2 + + store.RecordFailure(testIntervalKey, amt, capacity) + + forward := store.Get(testIntervalKey, capacity) + require.Equal(t, amt, forward.UpperFail) + require.Equal(t, intervalModeDepleted, forward.Mode) + require.EqualValues(t, 1, forward.Failures) + + // The failing amount and anything above it is now impossible, while a + // much smaller amount is still worth trying. + require.Zero(t, store.Probability(testIntervalKey, amt, capacity)) + require.Zero(t, store.Probability(testIntervalKey, amt+1, capacity)) + require.Greater( + t, store.Probability(testIntervalKey, amt/100, capacity), 0.0, + ) + + // A failure in one direction is evidence of available liquidity in the + // other, and is counted there as a success. + reverse := store.Get(testIntervalKey.Reverse(), capacity) + require.Equal(t, capacity-amt+1, reverse.LowerOK) + require.Equal(t, intervalModeRich, reverse.Mode) + require.EqualValues(t, 1, reverse.Successes) + + // A smaller failure tightens the bound, a larger one does not loosen + // it. + store.RecordFailure(testIntervalKey, amt/2, capacity) + require.Equal(t, amt/2, store.Get(testIntervalKey, capacity).UpperFail) + + store.RecordFailure(testIntervalKey, amt, capacity) + require.Equal(t, amt/2, store.Get(testIntervalKey, capacity).UpperFail) +} + +// TestIntervalStoreProbeThenFailure tests that a failure after a success leaves +// a bracketed interval rather than throwing one of the two observations away. +func TestIntervalStoreProbeThenFailure(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + capacity := testIntervalCapacity + + store.RecordProbe(testIntervalKey, capacity/10, capacity) + store.RecordFailure(testIntervalKey, capacity/2, capacity) + + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, capacity/10, interval.LowerOK) + require.Equal(t, capacity/2, interval.UpperFail) + require.Less(t, interval.Estimate, interval.UpperFail) + require.GreaterOrEqual(t, interval.Estimate, interval.LowerOK) + + // A failure at an amount we have already proven passes cannot leave the + // interval inverted. The upper bound is what gives way, because the + // lower one records something we watched succeed. + store.RecordFailure(testIntervalKey, capacity/20, capacity) + + interval = store.Get(testIntervalKey, capacity) + if interval.UpperFail != 0 { + require.Less(t, interval.LowerOK, interval.UpperFail) + } +} + +// TestIntervalStoreSettlement tests that a settlement shifts both directions of +// the interval rather than merely narrowing them, because the balance really +// has moved across the channel. +func TestIntervalStoreSettlement(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + capacity := testIntervalCapacity + amt := capacity / 10 + + // Prove a large amount passes, then settle a smaller one over it. + store.RecordProbe(testIntervalKey, capacity/2, capacity) + before := store.Get(testIntervalKey, capacity) + + store.RecordSettlement(testIntervalKey, amt, capacity) + after := store.Get(testIntervalKey, capacity) + + // The forward interval slides down by what left. + require.Equal(t, before.LowerOK-amt, after.LowerOK) + require.Equal(t, before.Estimate-amt, after.Estimate) + + // The reverse interval slides up by the same. + reverse := store.Get(testIntervalKey.Reverse(), capacity) + require.GreaterOrEqual(t, reverse.LowerOK, amt) + require.Equal(t, capacity-after.Estimate, reverse.Estimate) + + // A settlement of the whole capacity leaves nothing behind rather than + // wrapping around. + require.NoError(t, store.Clear(t.Context())) + store.RecordSettlement(testIntervalKey, capacity, capacity) + + drained := store.Get(testIntervalKey, capacity) + require.Zero(t, drained.Estimate) + require.Zero(t, drained.LowerOK) + require.Equal(t, intervalModeDepleted, drained.Mode) + + filled := store.Get(testIntervalKey.Reverse(), capacity) + require.Equal(t, capacity, filled.LowerOK) + require.Equal(t, intervalModeRich, filled.Mode) +} + +// TestIntervalStoreConfidence tests that confidence is a saturating latch that +// only ever rises with evidence. +func TestIntervalStoreConfidence(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + capacity := testIntervalCapacity + + require.Zero(t, store.Get(testIntervalKey, capacity).Confidence) + + store.RecordProbe(testIntervalKey, capacity/10, capacity) + probed := store.Get(testIntervalKey, capacity).Confidence + require.EqualValues(t, intervalProbeConfidence, probed) + + // A failure carries more weight than a probe, so it raises confidence. + store.RecordFailure(testIntervalKey, capacity/2, capacity) + failed := store.Get(testIntervalKey, capacity).Confidence + require.Greater(t, failed, probed) + + // A settlement latches at a lower level, which must not pull the latch + // back down. + store.RecordSettlement(testIntervalKey, capacity/100, capacity) + settled := store.Get(testIntervalKey, capacity).Confidence + require.Equal(t, failed, settled) +} + +// TestIntervalStoreIgnoresUninformativeObservations tests that observations the +// model cannot use are dropped rather than recorded as something they are not. +func TestIntervalStoreIgnoresUninformativeObservations(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + + // A zero amount says nothing. + store.RecordFailure(testIntervalKey, 0, testIntervalCapacity) + require.Zero(t, store.Len()) + + // Neither does an observation about a channel whose size we do not + // know, since every threshold in the model is a fraction of capacity. + store.RecordFailure(testIntervalKey, 100, 0) + require.Zero(t, store.Len()) + + // An amount larger than the capacity is clamped rather than dropped, + // because the capacity we path find against can be synthetic. + store.RecordFailure( + testIntervalKey, testIntervalCapacity*2, testIntervalCapacity, + ) + interval := store.Get(testIntervalKey, testIntervalCapacity) + require.Equal(t, testIntervalCapacity, interval.UpperFail) +} + +// TestIntervalStoreRestoreIsSoft tests the one property a restored belief must +// have. A bound written down before a restart describes a network that has had +// every chance to move on, and this model has no clock and no way to revise a +// bound except by attempting the amount. A restored upper bound that returned +// zero would therefore be permanent: the amount would never be tried again, so +// the evidence that would correct it could never arrive. +func TestIntervalStoreRestoreIsSoft(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + // Gather a hard bound the ordinary way, and read it back out the way a + // persistence layer would. + fresh := NewIntervalStore(0) + fresh.RecordFailure(testIntervalKey, amt, capacity) + require.Zero(t, fresh.Probability(testIntervalKey, amt, capacity)) + + saved := make(map[IntervalKey]LiquidityInterval) + fresh.ForEach(func(key IntervalKey, interval LiquidityInterval) { + saved[key] = interval + }) + require.Contains(t, saved, testIntervalKey) + + // Hand it to a store that has just started up. + restored := NewIntervalStore(0) + for key, interval := range saved { + restored.Restore(key, interval) + } + + interval := restored.Get(testIntervalKey, capacity) + require.True(t, interval.Restored) + require.Equal(t, saved[testIntervalKey].UpperFail, interval.UpperFail) + + // The bound survived, but it no longer says impossible, so the amount + // can be tried again and the belief can be corrected. + probability := restored.Probability(testIntervalKey, amt, capacity) + require.GreaterOrEqual(t, probability, intervalRestoredFloor) + require.Less(t, probability, 0.5) + + // It still says the amount is a bad bet, which is the whole point of + // keeping it: a restored bound outranks having no belief at all. + require.Less( + t, probability, + NewIntervalStore(0).Probability(testIntervalKey, amt, capacity), + ) + + // A restored lower bound is softened from the other side, so a channel + // that used to carry an amount is not trusted as if we had just watched + // it do so. + proven := NewIntervalStore(0) + proven.Restore(testIntervalKey, LiquidityInterval{ + Known: true, LowerOK: amt, Estimate: amt, Confidence: 1, + }) + + restoredHigh := proven.Probability(testIntervalKey, amt, capacity) + require.LessOrEqual(t, restoredHigh, intervalRestoredCeiling) + require.Less(t, restoredHigh, intervalProvenProbability) + + // Confidence is cut, because whatever stood behind the belief is at + // least one restart old. + require.Equal( + t, intervalRestoredConfidence, + proven.Get(testIntervalKey, capacity).Confidence, + ) +} + +// TestIntervalStoreFreshEvidenceBeatsRestored tests that a restored belief +// gives way to an observation this process made, on both sides of the channel. +func TestIntervalStoreFreshEvidenceBeatsRestored(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + store := NewIntervalStore(0) + store.Restore(testIntervalKey, LiquidityInterval{ + Known: true, UpperFail: amt, Confidence: 1, + }) + require.True(t, store.Get(testIntervalKey, capacity).Restored) + + // Watching the channel carry the amount replaces the restored belief + // outright, certainty and all. + store.RecordProbe(testIntervalKey, amt, capacity) + + interval := store.Get(testIntervalKey, capacity) + require.False(t, interval.Restored) + require.Equal(t, amt, interval.LowerOK) + require.EqualValues( + t, intervalProvenProbability, + store.Probability(testIntervalKey, amt, capacity), + ) + + // The reverse direction was written by the same observation, so it is + // no longer restored either. + require.False(t, store.Get(testIntervalKey.Reverse(), capacity).Restored) + + // A restore that arrives after we have seen the channel ourselves is + // ignored, since what we watched beats what we read back. + store.Restore(testIntervalKey, LiquidityInterval{ + Known: true, UpperFail: 1, + }) + require.False(t, store.Get(testIntervalKey, capacity).Restored) + require.Equal(t, amt, store.Get(testIntervalKey, capacity).LowerOK) +} + +// TestIntervalStoreEviction tests that the store stays inside its bound. +func TestIntervalStoreEviction(t *testing.T) { + t.Parallel() + + const maxEntries = 100 + + store := NewIntervalStore(maxEntries) + capacity := testIntervalCapacity + + for i := 0; i < maxEntries*4; i++ { + key := IntervalKey{ + ChanID: uint64(i), + From: route.Vertex{byte(i), byte(i >> 8)}, + To: route.Vertex{byte(i >> 8), byte(i)}, + } + + store.RecordFailure(key, capacity/2, capacity) + require.LessOrEqual(t, store.Len(), maxEntries+2) + } + + // The most recent observation survived the eviction. + final := maxEntries*4 - 1 + last := IntervalKey{ + ChanID: uint64(final), + From: route.Vertex{byte(final), byte(final >> 8)}, + To: route.Vertex{byte(final >> 8), byte(final)}, + } + require.True(t, store.Get(last, capacity).Known) + + require.NoError(t, store.Clear(t.Context())) + require.Zero(t, store.Len()) +} From a637ddbea02ed0fc898494280649d28d77f5e42a Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:17:43 -0700 Subject: [PATCH 3/9] routing: add a label-setting path finder over liquidity intervals In this commit, we add the search that turns the beliefs into a route. It walks backwards from the target the same way lnd's own path finder does, and it reuses the machinery that makes that walk correct: the edge unifier that picks a policy per node pair, the bandwidth hints that speak for our own channels, the fee and time lock limits, the onion payload accounting, and the feature validation of every node it would route through. Two things differ. The cost of a hop is the negative log of its probability plus terms for fee, depth and how much of a channel the payment would fill, rather than a fee divided through by a probability. Additivity is what makes the search tractable, and it is far gentler on a route the model is merely unsure about. The other difference is that a node keeps a bounded set of Pareto-incomparable labels rather than a single best distance. Because the search runs backwards and fees accrue as it goes, a route that is cheaper but carries a larger amount is genuinely incomparable to one that is dearer and carries less, since the larger amount may be refused further upstream. The units of the fee term decide whether a fee budget can reach the search at all, which is worth stating plainly because it is easy to get wrong. Converting a fee at a rate proportional to the amount reads as a willingness to pay a fifth of the payment for one factor of e in reliability, and no budget anybody sets is within reach of that, so the term never binds. A payment carrying a budget therefore prices fees against the budget itself, half of what remains, bounded at both ends and absolute rather than relative, so the ceiling tightens as the payment grows. A payment with no budget has nothing to derive a rate from and falls back to the amount relative form. Two smaller rules follow from the same concern. A node keeps the cheapest label it holds whatever that label scores, but only when the payment has a budget that could bind, since a slot spent on price is a slot wasted for a payment choosing on reliability alone. And the search is bounded on hops, labels per node and total expansions, because it is more expensive than a single distance Dijkstra by construction. --- routing/interval_config.go | 127 +++++ routing/interval_pathfind.go | 948 +++++++++++++++++++++++++++++++++++ 2 files changed, 1075 insertions(+) create mode 100644 routing/interval_config.go create mode 100644 routing/interval_pathfind.go diff --git a/routing/interval_config.go b/routing/interval_config.go new file mode 100644 index 00000000000..8937c9ba8f0 --- /dev/null +++ b/routing/interval_config.go @@ -0,0 +1,127 @@ +package routing + +import "github.com/lightningnetwork/lnd/lnwire" + +const ( + // DefaultPaymentRouter selects lnd's production routing stack, which is + // Dijkstra over a probability estimator with mission control behind it, + // and reactive halving of the shard amount when no route is found. + DefaultPaymentRouter = "default" + + // IntervalPaymentRouter selects the interval router, which replaces + // mission control with a per directed channel liquidity interval and + // plans the shard amount and the route together. + IntervalPaymentRouter = "interval" +) + +// Search bounds of the interval router. The label setting search is more +// expensive than a single distance Dijkstra by construction, since a node may +// keep several incomparable labels and every rung of the shard ladder is priced +// with its own search, so each of these is a real ceiling rather than a +// formality. +const ( + // DefaultIntervalMaxRouteHops is the longest route the search will + // build. It is well above the twenty hop limit an onion can express, + // and the payload size check is what actually binds first. + DefaultIntervalMaxRouteHops = 24 + + // DefaultIntervalMaxLabels is how many Pareto-incomparable labels a + // single node may keep. + DefaultIntervalMaxLabels = 24 + + // DefaultIntervalSearchLimit is how many node expansions a single + // search may perform before it gives up and returns the best route it + // has, if any. + DefaultIntervalSearchLimit = 120000 + + // DefaultIntervalAttemptLimit is how many HTLCs one payment may spend + // before the session stops handing out routes. The payment lifecycle + // has its own bounds, the payment timeout and the part limit among + // them; this one exists so that a session which believes it can always + // find one more route cannot spin forever. + DefaultIntervalAttemptLimit = 80 + + // DefaultIntervalMaxShards caps how many pieces the shard ladder will + // consider cutting a payment into, independently of the part limit the + // payment itself carries. + DefaultIntervalMaxShards = 64 + + // DefaultIntervalMaxLadderRungs caps how many candidate shard sizes are + // priced for a single route request. Every rung costs a full search, so + // this is the knob that decides what one call to RequestRoute costs. + // The rungs are enumerated in order of how much they are worth pricing, + // so a cap keeps the most informative ones. + DefaultIntervalMaxLadderRungs = 16 +) + +// IntervalConfig holds the tunables of the interval router. The defaults are +// the values the algorithm was validated with, and they are exposed here so +// that they can be moved in a test rather than because an operator is expected +// to turn them. +type IntervalConfig struct { + // MaxRouteHops is the longest route the search will build. + MaxRouteHops uint16 + + // MaxLabels is how many incomparable labels a node may keep. + MaxLabels int + + // SearchLimit bounds the number of expansions of a single search. + SearchLimit int + + // AttemptLimit bounds the number of HTLCs one payment may spend. + AttemptLimit uint32 + + // MaxShards caps the number of pieces the shard ladder considers. + MaxShards uint32 + + // MaxLadderRungs caps how many candidate shard sizes are priced for a + // single route request. + MaxLadderRungs int + + // MinShardAmt is the smallest shard the router will send. Below it, a + // payment that still cannot be routed is given up on rather than cut + // any finer. + MinShardAmt lnwire.MilliSatoshi +} + +// DefaultIntervalConfig returns the configuration the interval router was +// validated with. +func DefaultIntervalConfig() IntervalConfig { + return IntervalConfig{ + MaxRouteHops: DefaultIntervalMaxRouteHops, + MaxLabels: DefaultIntervalMaxLabels, + SearchLimit: DefaultIntervalSearchLimit, + AttemptLimit: DefaultIntervalAttemptLimit, + MaxShards: DefaultIntervalMaxShards, + MaxLadderRungs: DefaultIntervalMaxLadderRungs, + MinShardAmt: DefaultShardMinAmt, + } +} + +// fillDefaults replaces any unset field with its default, so that a zero valued +// config is usable. +func (c *IntervalConfig) fillDefaults() { + defaults := DefaultIntervalConfig() + + if c.MaxRouteHops == 0 { + c.MaxRouteHops = defaults.MaxRouteHops + } + if c.MaxLabels <= 0 { + c.MaxLabels = defaults.MaxLabels + } + if c.SearchLimit <= 0 { + c.SearchLimit = defaults.SearchLimit + } + if c.AttemptLimit == 0 { + c.AttemptLimit = defaults.AttemptLimit + } + if c.MaxShards == 0 { + c.MaxShards = defaults.MaxShards + } + if c.MaxLadderRungs <= 0 { + c.MaxLadderRungs = defaults.MaxLadderRungs + } + if c.MinShardAmt == 0 { + c.MinShardAmt = defaults.MinShardAmt + } +} diff --git a/routing/interval_pathfind.go b/routing/interval_pathfind.go new file mode 100644 index 00000000000..db9450882e3 --- /dev/null +++ b/routing/interval_pathfind.go @@ -0,0 +1,948 @@ +package routing + +import ( + "container/heap" + "context" + "math" + + sphinx "github.com/lightningnetwork/lightning-onion" + "github.com/lightningnetwork/lnd/feature" + "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +// The weights below turn a candidate hop into a cost. The stock path finder +// minimizes fee plus a time lock penalty, divided through by the route +// probability; this one minimizes an additive score whose dominant term is the +// negative log of the hop probability. Additivity is what makes a label setting +// search over the graph tractable, and it is far gentler on a low probability +// route than dividing by a probability is. +const ( + // intervalFeeWeight sets the fee sensitivity of the search when the + // payment carries no fee budget. See intervalFeePenalty for what this + // number means and for the units it is expressed in, which is the part + // of this cost function most worth understanding. + intervalFeeWeight = 5.0 + + // intervalHopBase and intervalHopGrowth price adding another hop. The + // penalty grows with depth, so a long route becomes progressively more + // expensive rather than paying a flat toll per hop. + intervalHopBase = 0.045 + intervalHopGrowth = 0.003 + + // intervalCapacityKnee is the channel utilization at which the capacity + // penalty starts, and intervalCapacityWeight is its weight at full + // utilization. This steers the search away from channels the payment + // would nearly fill even when nothing is known against them. + intervalCapacityKnee = 0.70 + intervalCapacityWeight = 0.30 + + // intervalLabelAmountWeight and intervalLabelHopWeight rank labels + // against each other when a node holds more of them than it is allowed + // to keep. The worst by this rank is the one evicted. + intervalLabelAmountWeight = 0.10 + intervalLabelHopWeight = 0.014 + + // intervalBudgetShare is the fraction of a payment's remaining fee + // budget it will spend to buy one nat of reliability. Half means a + // payment will pay up to half of what it has left to raise the + // probability of a route by a factor of e, which leaves the other half + // for the hops that follow. + intervalBudgetShare = 2.0 + + // intervalMinFeePrice and intervalMaxFeePrice bound the budget derived + // exchange rate, in millisatoshis per nat. The floor keeps a payment + // with almost nothing left from refusing to pay any fee at all, since a + // route it can afford is still better than no route. The ceiling keeps a + // payment with a very large budget from treating fees as free. + intervalMinFeePrice = 30_000.0 + intervalMaxFeePrice = 420_000.0 +) + +// intervalFeeRate says how a payment converts a fee into the nats its search +// score is denominated in. It has two fields on purpose, because the two +// questions it answers have different answers and confusing them is a bug we +// have already made once. +// +// Whether the payment has a budget at all is latched from the limit the payment +// was created with, and never changes for as long as the payment lives. How +// dearly a payment with a budget prices reliability comes from what that limit +// has left, which shrinks as shards commit and which the lifecycle recomputes +// before every route request. +// +// Inferring the first from the second is what goes wrong. lnd hands a session +// the remaining budget, so a payment with no limit carries the sentinel only +// until its first shard pays a fee; from the second shard on it carries the +// sentinel minus that fee, which is not the sentinel. Read as a classification +// that says budgeted, and every unbudgeted payment that splits silently starts +// pricing fees against a budget nobody set. +type intervalFeeRate struct { + // budgeted is latched from the payment's own fee limit and decides which + // branch of penalty applies. + budgeted bool + + // price is how many millisatoshis of fee buy one nat of log + // probability, derived from what the budget has left. It is only + // meaningful when budgeted is set. + price float64 +} + +// newIntervalFeeRate builds the rate for one route request. The caller passes +// the latched classification and the budget remaining right now, which is the +// only combination that keeps the two apart. +func newIntervalFeeRate(budgeted bool, + remaining lnwire.MilliSatoshi) intervalFeeRate { + + if !budgeted { + return intervalFeeRate{} + } + + return intervalFeeRate{ + budgeted: true, + price: intervalBudgetPrice(remaining), + } +} + +// penalty converts a fee into nats. A payment with a budget pays the rate its +// budget sets; one without pays a fixed fraction of the amount it is sending. +// +// The unbudgeted branch is written as the one expression it has always been, +// weight times fee over the amount, rather than as a division by the reciprocal +// of that. The two agree in exact arithmetic and they do not agree in floating +// point: over the amounts a real corpus holds they differ by one unit in the +// last place about a quarter of the time, because dividing the amount first +// rounds once and dividing the fee by that result rounds again. +// +// One unit in the last place is normally beneath notice. It is not beneath +// notice here, because the frontier compares these scores exactly, both to +// decide whether one label dominates another and to decide which label to +// evict when a node is full. A tie that used to break one way breaks the other, +// a different route comes back, and the payment goes somewhere else. When that +// change was made by accident it was worth 0.032 of objective on one tier, all +// of it in success. So the rule for this branch is bit identity with what came +// before, not algebraic identity, and the way to keep that is to leave the +// expression alone. +func (r intervalFeeRate) penalty(fee float64, amt lnwire.MilliSatoshi, + weight float64) float64 { + + if r.budgeted { + return fee / r.price + } + + return weight * fee / math.Max(float64(amt), 1) +} + +// intervalBudgetPrice returns how many millisatoshis of fee a payment will +// trade for one nat of log probability, given what its budget has left. It is +// the exchange rate between the two things the search is minimizing, and it is +// where the units of the cost function are decided. +// +// The score this rate feeds is denominated in nats: a hop contributes the +// negative log of its probability, plus its fee converted through this rate. +// Which way that conversion runs turns out to decide whether a fee budget can +// ever influence the search at all. +// +// The routers this design came from converted fees at a rate proportional to +// the amount being sent, k times fee over amount with k around 5. Read as a +// price, that is a willingness to pay amount/k for one nat, which is a fifth +// of the payment. No realistic fee budget is anywhere near a fifth of it, +// so the fee term never binds and the search prices reliability as though money +// were free. Measurement bore that out: those routers walk into fee budgets +// they cannot see, while lnd, whose path finding prices fees in absolute +// millisatoshis, never violates one. +// +// So the budget sets the rate. A payment with 10,000 millisatoshis left will +// pay 5,000 of them for one nat, which is a price a real route can exceed, and +// the search starts declining expensive reliability on its own rather than +// discovering the limit when the route is rejected. The rate is absolute, so it +// tightens in relative terms as the payment grows, which is the right +// direction: a fee budget quoted in parts per million bites hardest in absolute +// terms on the largest payments. +// +// NOTE: this must only be called for a payment that has a budget. It does not +// classify, and handing it the remainder of an absent budget would produce a +// perfectly plausible looking rate for a limit nobody set. +func intervalBudgetPrice(remaining lnwire.MilliSatoshi) float64 { + price := float64(remaining) / intervalBudgetShare + + return math.Min( + math.Max(price, intervalMinFeePrice), intervalMaxFeePrice, + ) +} + +// intervalBudgeted reports whether a payment carries a fee budget that a route +// could exceed. Anything short of the sentinel is a real limit. +// +// NOTE: this belongs on the limit a payment was created with, and nowhere else. +// Applied to a remaining budget it answers a different question and gets it +// wrong, because a limit that has had fees subtracted from it no longer looks +// like the sentinel even when there was never a limit to begin with. +func intervalBudgeted(feeLimit lnwire.MilliSatoshi) bool { + return feeLimit != lnwire.MaxMilliSatoshi +} + +// intervalLabel is one Pareto-incomparable way of reaching the target from a +// node. The stock path finder keeps a single best distance per node, which is +// enough when the only thing being minimized is a scalar cost. Here it is not +// enough: because the search runs backwards and fees accrue along the way, a +// route that is cheaper but carries a larger amount is genuinely incomparable +// to one that is dearer but carries less, since the larger amount may be +// refused further upstream. A label carries all three of the quantities that +// decide that comparison. +type intervalLabel struct { + // node is the node this label describes a route from. + node route.Vertex + + // netAmountReceived is the amount this node needs to receive, with its + // own inbound fee already subtracted. + netAmountReceived lnwire.MilliSatoshi + + // outboundFee is the fee this node charges for the hop it forwards + // over, which is needed to keep a negative inbound fee from taking a + // node's total fee below zero. + outboundFee lnwire.MilliSatoshi + + // incomingCltv is the expiry the incoming HTLC to this node carries. + incomingCltv int32 + + // routingInfoSize is the accumulated onion payload size of the route + // from this node onwards. + routingInfoSize uint64 + + // score is the accumulated cost of the route from this node to the + // target, and is what the search minimizes. + score float64 + + // risk is the accumulated negative log probability of the route, kept + // apart from the score so that the caller can price a shard on + // probability alone. + risk float64 + + // hops is the number of hops from this node to the target. + hops uint16 + + // edge is the hop out of this node, and child is the label it leads to. + edge *unifiedEdge + child *intervalLabel + + // active is cleared when this label is dominated by another, at which + // point any copy of it still sitting in the heap is skipped. + active bool +} + +// contains reports whether the given node already appears on the route this +// label describes, which is how the search avoids walking in circles. +func (l *intervalLabel) contains(node route.Vertex) bool { + for current := l; current != nil; current = current.child { + if current.node == node { + return true + } + } + + return false +} + +// rank scores a label for eviction. It folds the amount and the hop count into +// the score so that the label a node gives up is the one least likely to be +// part of a good route. +func (l *intervalLabel) rank(deliver lnwire.MilliSatoshi) float64 { + amountRatio := math.Max( + float64(l.netAmountReceived)/float64(deliver), 1, + ) + + return l.score + intervalLabelAmountWeight*math.Log(amountRatio) + + intervalLabelHopWeight*float64(l.hops) +} + +// intervalHeap is a min-heap of labels ordered by score. +type intervalHeap []*intervalLabel + +func (h intervalHeap) Len() int { return len(h) } + +func (h intervalHeap) Less(i, j int) bool { return h[i].score < h[j].score } + +func (h intervalHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } + +func (h *intervalHeap) Push(value any) { + *h = append(*h, value.(*intervalLabel)) +} + +func (h *intervalHeap) Pop() any { + old := *h + last := len(old) - 1 + item := old[last] + old[last] = nil + *h = old[:last] + + return item +} + +// intervalFrontier holds the labels a node has kept, bounded in size. +type intervalFrontier struct { + labels map[route.Vertex][]*intervalLabel + maxLabels int + + // keepCheapest protects the cheapest label a node holds from eviction. + // It is set for a payment that carries a fee budget and cleared for one + // that does not, which is a distinction the measurements insisted on. + // See insert for what goes wrong when it is set unconditionally. + keepCheapest bool +} + +// insert files a label under its node, dropping it if an existing label already +// dominates it and dropping any existing labels it dominates in turn. It +// reports whether the label was kept. +func (f *intervalFrontier) insert(label *intervalLabel, + deliver lnwire.MilliSatoshi) bool { + + existing := f.labels[label.node] + + // A label is dominated only when another label is no worse on all three + // of score, amount and hop count. Anything less than that is a genuine + // trade-off and both are kept. + for _, old := range existing { + if old.active && + old.score <= label.score+1e-12 && + old.netAmountReceived <= label.netAmountReceived && + old.hops <= label.hops { + + return false + } + } + + kept := make([]*intervalLabel, 0, len(existing)+1) + for _, old := range existing { + if !old.active { + continue + } + + if label.score <= old.score+1e-12 && + label.netAmountReceived <= old.netAmountReceived && + label.hops <= old.hops { + + old.active = false + + continue + } + + kept = append(kept, old) + } + + kept = append(kept, label) + if len(kept) > f.maxLabels { + // When the payment carries a fee budget, the cheapest label a + // node holds is kept whatever its score, because it is the one + // that survives if the budget binds. The amount a label needs to + // receive is the fee it has accumulated plus the amount being + // delivered, so the smallest of those is the cheapest route out + // of this node. Without this the frontier fills with reliable + // expensive labels and a payment that cannot afford them is left + // with nothing to fall back to. + // + // When there is no budget the protection is dropped, because a + // label kept for a budget that does not exist displaces a better + // label for the payment actually being made. Measurement was + // blunt about it: keeping the cheapest label unconditionally + // cost 0.032 of objective on the out of distribution tier, all + // of it in success rather than attempts, and the same shape + // showed up on the unbudgeted economic control. The budgeted + // tiers are where the keep earns its place, so that is where it + // applies. + protected := -1 + if f.keepCheapest { + protected = 0 + for i := 1; i < len(kept); i++ { + if kept[i].netAmountReceived < + kept[protected].netAmountReceived { + + protected = i + } + } + } + + worst := -1 + worstRank := 0.0 + for i := range kept { + if i == protected { + continue + } + + if rank := kept[i].rank(deliver); worst < 0 || + rank > worstRank { + + worst = i + worstRank = rank + } + } + + // If the label we were handed is the worst of the set, there is + // no room for it. + if worst < 0 || kept[worst] == label { + return false + } + + kept[worst].active = false + kept = append(kept[:worst], kept[worst+1:]...) + } + + label.active = true + f.labels[label.node] = kept + + return true +} + +// intervalEdgeProbability is the callback the search uses to price a hop. It is +// handed the directed channel the hop would use, the amount that would be sent +// over it, and the channel's capacity. +type intervalEdgeProbability func(key IntervalKey, amt, + capacity lnwire.MilliSatoshi) float64 + +// intervalPathParams gathers everything the interval path finder needs for one +// search. It mirrors the arguments of the stock path finder, with the +// probability source replaced by one that reads the interval store rather than +// mission control. +type intervalPathParams struct { + // graph carries the channel graph, the ephemeral edges and the + // bandwidth hints for our own channels. + graph *graphParams + + // restrictions are the constraints the route must respect, and are the + // same ones the stock path finder is given. + restrictions *RestrictParams + + // cfg holds the search bounds. + cfg *IntervalConfig + + // probability prices a single hop. + probability intervalEdgeProbability + + // self is our own node, source the node the route starts at and target + // the node it ends at. + self, source, target route.Vertex + + // amt is the amount to deliver to the target. + amt lnwire.MilliSatoshi + + // finalHtlcExpiry is the absolute expiry height of the final hop. + finalHtlcExpiry int32 + + // cache holds the graph reads that do not depend on the amount, so that + // every rung of a shard ladder pays for them once between them rather + // than once each. + cache *intervalGraphCache + + // feeRate says how this payment converts fees into the nats the score is + // denominated in. + feeRate intervalFeeRate +} + +// intervalGraphCache holds what a search learns from the graph that does not +// change with the amount being routed. It is shared by every search of a single +// call to RequestRoute, which holds a graph session open across all of them, so +// the graph cannot move underneath it. +type intervalGraphCache struct { + // unifiers holds the channels into a node, keyed by the node they come + // from. Building these is the part of the search that touches the graph + // database, so caching them is what makes pricing a whole shard ladder + // affordable. + unifiers map[route.Vertex]map[route.Vertex]*edgeUnifier + + // features holds the validated feature vector of a node, with a nil + // entry meaning the node cannot be routed through. + features map[route.Vertex]*lnwire.FeatureVector +} + +// newIntervalGraphCache builds an empty cache. +func newIntervalGraphCache() *intervalGraphCache { + return &intervalGraphCache{ + unifiers: make( + map[route.Vertex]map[route.Vertex]*edgeUnifier, + ), + features: make(map[route.Vertex]*lnwire.FeatureVector), + } +} + +// siblingCount returns how many channels connect the given directed pair, or +// zero when the search never looked at that pair. It is what decides whether an +// observation about a hop is allowed to name a channel. +func (c *intervalGraphCache) siblingCount(from, to route.Vertex) int { + unifiers, ok := c.unifiers[to] + if !ok { + return 0 + } + + unifier, ok := unifiers[from] + if !ok { + return 0 + } + + return len(unifier.edges) +} + +// findIntervalPath searches for a route from source to target able to deliver +// amt, scoring hops with the interval belief model. Like the stock path finder +// it searches backwards from the target so that fees and amounts accumulate in +// the direction they are actually paid, and it returns the path in forward +// order along with the negative log probability of the route. +// +// The search is label setting rather than shortest path: each node keeps a +// bounded set of Pareto-incomparable ways of reaching the target, so a route +// carrying a smaller amount at a higher cost survives alongside a cheaper one +// that carries more. +func findIntervalPath(ctx context.Context, p *intervalPathParams) ( + []*unifiedEdge, float64, error) { + + features, err := intervalDestFeatures(ctx, p) + if err != nil { + return nil, 0, err + } + + // Set up outgoing channel map for quicker access. + var outgoingChanMap map[uint64]struct{} + if len(p.restrictions.OutgoingChannelIDs) > 0 { + outgoingChanMap = make(map[uint64]struct{}) + for _, outChan := range p.restrictions.OutgoingChannelIDs { + outgoingChanMap[outChan] = struct{}{} + } + } + + // Build the reverse lookup of the ephemeral edges, since the search + // runs from the target back towards us. + additionalEdgesWithSrc := make(map[route.Vertex][]*edgePolicyWithSource) + for vertex, edges := range p.graph.additionalEdges { + if vertex == p.self { + continue + } + + for _, edge := range edges { + policy := edge.EdgePolicy() + toVertex := policy.ToNodePubKey() + + additionalEdgesWithSrc[toVertex] = append( + additionalEdgesWithSrc[toVertex], + &edgePolicyWithSource{ + sourceNode: vertex, + edge: edge, + }, + ) + } + } + + lastHopSize, err := lastHopPayloadSize( + p.restrictions, p.finalHtlcExpiry, p.amt, + ) + if err != nil { + return nil, 0, err + } + + // The search starts at the target, which needs to receive the full + // amount and charges nothing. + root := &intervalLabel{ + node: p.target, + netAmountReceived: p.amt, + incomingCltv: p.finalHtlcExpiry, + routingInfoSize: lastHopSize, + active: true, + } + + queue := &intervalHeap{root} + frontier := &intervalFrontier{ + labels: map[route.Vertex][]*intervalLabel{}, + maxLabels: p.cfg.MaxLabels, + // Protecting the cheapest label is worth a slot for a payment + // whose budget could bind, and a cost for one with no budget to + // bind, so the answer is whether it carries a limit at all. + keepCheapest: p.feeRate.budgeted, + } + + // Calculate the absolute cltv limit. Use uint64 to prevent an overflow + // if the cltv limit is MaxUint32. + absoluteCltvLimit := uint64(p.restrictions.CltvLimit) + + uint64(p.finalHtlcExpiry) + + cache := p.cache + if cache == nil { + cache = newIntervalGraphCache() + } + + search := &intervalSearch{ + params: p, + frontier: frontier, + queue: queue, + outgoingChanMap: outgoingChanMap, + additionalEdges: additionalEdgesWithSrc, + absoluteCltvLimit: absoluteCltvLimit, + cache: cache, + } + + best, err := search.run(ctx) + if err != nil { + return nil, 0, err + } + if best == nil { + return nil, 0, errNoPathFound + } + + // Unravel the label chain into a forward ordered path. + var pathEdges []*unifiedEdge + for current := best; current != nil && current.edge != nil; { + pathEdges = append(pathEdges, current.edge) + current = current.child + } + + // The final hop's features are the ones we validated above, which may + // come from the invoice rather than from the graph. + pathEdges[len(pathEdges)-1].policy.ToNodeFeatures = features + + return pathEdges, best.risk, nil +} + +// intervalDestFeatures resolves and validates the feature vector of the payment +// destination, exactly as the stock path finder does. +func intervalDestFeatures(ctx context.Context, p *intervalPathParams) ( + *lnwire.FeatureVector, error) { + + features := p.restrictions.DestFeatures + if features == nil { + var err error + features, err = p.graph.graph.FetchNodeFeatures(ctx, p.target) + if err != nil { + return nil, err + } + } + + if err := feature.ValidateRequired(features); err != nil { + log.Warnf("Interval pathfinding destination features: %v", err) + + return nil, errUnknownRequiredFeature + } + + if err := feature.ValidateDeps(features); err != nil { + log.Warnf("Interval pathfinding destination features: %v", err) + + return nil, errMissingDependentFeature + } + + if p.restrictions.PaymentAddr.IsSome() && + !features.HasFeature(lnwire.PaymentAddrOptional) { + + return nil, errNoPaymentAddr + } + + return features, nil +} + +// intervalSearch holds the mutable state of one path finding run. +type intervalSearch struct { + params *intervalPathParams + frontier *intervalFrontier + queue *intervalHeap + + outgoingChanMap map[uint64]struct{} + additionalEdges map[route.Vertex][]*edgePolicyWithSource + absoluteCltvLimit uint64 + + // cache holds the graph reads shared with every other search of the + // same route request. + cache *intervalGraphCache + + expansions int +} + +// run walks the graph until it pops a label at the source node, exhausts the +// queue, or runs out of its expansion budget. +func (s *intervalSearch) run(ctx context.Context) (*intervalLabel, error) { + p := s.params + + for s.queue.Len() != 0 { + label := heap.Pop(s.queue).(*intervalLabel) + if !label.active { + continue + } + + // Reaching the source means we have a complete route. Because + // the queue is ordered by score, the first one we pop is the + // cheapest. The hop count guard is what keeps a payment to + // ourselves from terminating on its own starting label. + if label.node == p.source && label.hops > 0 { + return label, nil + } + + if label.hops >= p.cfg.MaxRouteHops { + continue + } + + s.expansions++ + if s.expansions > p.cfg.SearchLimit { + log.Debugf("Interval pathfinding hit its expansion "+ + "budget of %v", p.cfg.SearchLimit) + + break + } + + if err := s.expand(ctx, label); err != nil { + return nil, err + } + } + + return nil, nil +} + +// expand walks every channel into the label's node and files the labels they +// produce. +func (s *intervalSearch) expand(ctx context.Context, + label *intervalLabel) error { + + p := s.params + + unifiers, err := s.incomingEdges(label.node) + if err != nil { + return err + } + + routeToSelf := p.source == p.target + + for fromNode, unifier := range unifiers { + // The target is where the search started, so walking back into + // it would close a loop. The one exception is a payment to + // ourselves, which is a loop by construction. + if !routeToSelf && fromNode == p.target { + continue + } + + // Apply the last hop restriction if one is set. + if p.restrictions.LastHop != nil && label.node == p.target && + fromNode != *p.restrictions.LastHop { + + continue + } + + // The source node is always allowed, since the search stops + // there; anything else already on the route would be a cycle. + if fromNode != p.source && label.contains(fromNode) { + continue + } + + edge := unifier.getEdge( + label.netAmountReceived, p.graph.bandwidthHints, + label.outboundFee, + ) + if edge == nil { + continue + } + + features, err := s.nodeFeatures(ctx, fromNode) + if err != nil { + return err + } + if features == nil { + continue + } + + // How many channels this pair has decides whether an + // observation about the hop can name one of them. + s.processEdge(fromNode, edge, label, len(unifier.edges)) + } + + return nil +} + +// processEdge prices one candidate hop and files the label it produces if the +// hop respects every restriction and is not dominated by a label the node +// already holds. +func (s *intervalSearch) processEdge(fromNode route.Vertex, edge *unifiedEdge, + label *intervalLabel, siblings int) { + + p := s.params + + // Calculate the inbound fee charged by the node we are walking back + // from, keeping its total fee from going negative. + inboundFee := edge.inboundFees.CalcFee(label.netAmountReceived) + if minInboundFee := -int64(label.outboundFee); inboundFee < + minInboundFee { + + inboundFee = minInboundFee + } + + // This is the amount the candidate node would have to send onwards. + amountToSend := label.netAmountReceived + + lnwire.MilliSatoshi(inboundFee) + + // Refuse to build a route whose accumulated fee runs past the budget. + totalFee := int64(amountToSend) - int64(p.amt) + if totalFee > 0 && + lnwire.MilliSatoshi(totalFee) > p.restrictions.FeeLimit { + + return + } + + probability := p.probability( + intervalScopeKey(IntervalKey{ + ChanID: edge.policy.ChannelID, + From: fromNode, + To: label.node, + }, siblings), + amountToSend, lnwire.NewMSatFromSatoshis(edge.capacity), + ) + if probability <= 0 { + return + } + + // The source node has no predecessor to charge a fee or a time lock. + var ( + timeLockDelta uint16 + outboundFee int64 + ) + if fromNode != p.source { + outboundFee = int64(edge.policy.ComputeFee(amountToSend)) + timeLockDelta = edge.policy.TimeLockDelta + } + + incomingCltv := label.incomingCltv + int32(timeLockDelta) + if uint64(incomingCltv) > s.absoluteCltvLimit { + return + } + + // Refuse to build an onion the network will not carry. + routingInfoSize := label.routingInfoSize + if fromNode != p.source { + if edge.hopPayloadSizeFn == nil { + log.Criticalf("No payload size function available for "+ + "edge=%v: %v", edge, ErrNoPayLoadSizeFunc) + + return + } + + routingInfoSize += edge.hopPayloadSizeFn( + amountToSend, uint32(label.incomingCltv), + edge.policy.ChannelID, + ) + } + if routingInfoSize > sphinx.MaxRoutingPayloadSize { + return + } + + // With the hop accepted, price it. The dominant term is the negative + // log of the probability, which is what makes the cost additive over + // hops in the first place. + signedFee := inboundFee + outboundFee + fee := float64(0) + if signedFee > 0 { + fee = float64(signedFee) + } + + edgeRisk := -math.Log(probability) + feePenalty := p.feeRate.penalty(fee, p.amt, intervalFeeWeight) + hopPenalty := intervalHopBase + + intervalHopGrowth*float64(label.hops) + + capacityPenalty := float64(0) + capacity := lnwire.NewMSatFromSatoshis(edge.capacity) + if capacity > 0 { + ratio := float64(amountToSend) / float64(capacity) + if ratio > intervalCapacityKnee { + over := (ratio - intervalCapacityKnee) / + (1 - intervalCapacityKnee) + capacityPenalty = intervalCapacityWeight * over * over + } + } + + candidate := &intervalLabel{ + node: fromNode, + netAmountReceived: amountToSend + lnwire.MilliSatoshi(outboundFee), + outboundFee: lnwire.MilliSatoshi(outboundFee), + incomingCltv: incomingCltv, + routingInfoSize: routingInfoSize, + score: label.score + edgeRisk + feePenalty + hopPenalty + + capacityPenalty, + risk: label.risk + edgeRisk, + hops: label.hops + 1, + edge: edge, + child: label, + } + + // A label at the source is a finished route, so it goes straight onto + // the queue rather than into a frontier it would never be expanded + // from. + if fromNode == p.source { + candidate.active = true + heap.Push(s.queue, candidate) + + return + } + + if !s.frontier.insert(candidate, p.amt) { + return + } + + heap.Push(s.queue, candidate) +} + +// incomingEdges returns the channels into the given node, keyed by the node +// they come from, building and caching them on first use. +func (s *intervalSearch) incomingEdges(node route.Vertex) ( + map[route.Vertex]*edgeUnifier, error) { + + if cached, ok := s.cache.unifiers[node]; ok { + return cached, nil + } + + p := s.params + + // The exit hop does not charge an inbound fee. + isExitHop := node == p.target + + u := newNodeEdgeUnifier(p.self, node, !isExitHop, s.outgoingChanMap) + if err := u.addGraphPolicies(p.graph.graph); err != nil { + return nil, err + } + + // Fold in any ephemeral edges that lead to this node. Hop hints carry + // no capacity, so we assume a large one, the same way the stock path + // finder does. + for _, reverseEdge := range s.additionalEdges[node] { + u.addPolicy( + reverseEdge.sourceNode, reverseEdge.edge.EdgePolicy(), + models.InboundFee{}, fakeHopHintCapacity, + reverseEdge.edge.IntermediatePayloadSize, + reverseEdge.edge.BlindedPayment(), + ) + } + + s.cache.unifiers[node] = u.edgeUnifiers + + return u.edgeUnifiers, nil +} + +// nodeFeatures returns the validated feature vector of a node, or nil if the +// node cannot be routed through. +func (s *intervalSearch) nodeFeatures(ctx context.Context, + node route.Vertex) (*lnwire.FeatureVector, error) { + + if cached, ok := s.cache.features[node]; ok { + return cached, nil + } + + features, err := s.params.graph.graph.FetchNodeFeatures(ctx, node) + if err != nil { + return nil, err + } + + // Do not route through nodes that require features we do not know, or + // that fail to set their transitive dependencies. + if err := feature.ValidateRequired(features); err != nil { + s.cache.features[node] = nil + + return nil, nil + } + if err := feature.ValidateDeps(features); err != nil { + s.cache.features[node] = nil + + return nil, nil + } + + s.cache.features[node] = features + + return features, nil +} From d8d93ec7db094109111c782a8cf61eb5dfb39b9b Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:18:30 -0700 Subject: [PATCH 4/9] routing: add the interval payment session In this commit, we add the payment session that owns route selection and splitting together. The session lnd ships does not choose a shard size: it asks path finding for the whole remaining amount and halves it when nothing comes back, so every split is a reaction to a failure and every shard is a power-of-two fraction. This one enumerates a ladder of candidate shard sizes, finds a route for each, and keeps the best pairing of the two, which means it can split before it has failed at all and can pick a shard sized to fit just under a bound it discovered two attempts ago. The ladder draws on four sources: the whole remaining amount and the smallest shard that could still finish the payment, the amounts this payment has already proven do not fit divided down until they do, the even divisions, and the halving chain. The second is what makes the ladder a function of the beliefs rather than of the amount alone, and it is enumerated early because every rung costs a full search and the ladder is capped. The session layers its own experience over the node wide store. A channel it watched refuse an amount is discounted by how much smaller the retry is, on a ladder that replaces blacklisting and waiting for a penalty to decay. A channel that rejected an HTLC on policy grounds is stepped around for the rest of the payment, because the policy we hold for it is stale and a retry would be built from the same stale policy. None of that outlives the payment, while everything written to the store does. Two things the session does that the amounts alone would not tell you. It counts the HTLCs it already has in flight when pricing a hop, since a channel holding some of ours must have had that much plus the new shard available when we last looked, and it releases those holds on three separate triggers so that one can never outlive its HTLC. And when a failure cannot be attributed to a hop, it narrows the suspects by elimination rather than penalizing the whole route: one suspect left is a certainty, none at all is a contradiction, and several share the blame in proportion to how many there are. Payments the router does not serve are handed to the stock session instead, so that turning it on cannot make a payment unroutable that would otherwise have gone through. Today that means payments to blinded paths, where there is no directed channel to key a belief on. --- routing/interval_budget_test.go | 497 +++++++++++ routing/interval_inflight_test.go | 384 ++++++++ routing/interval_parallel_test.go | 290 ++++++ routing/interval_session.go | 1327 ++++++++++++++++++++++++++++ routing/interval_session_source.go | 94 ++ routing/interval_session_test.go | 661 ++++++++++++++ 6 files changed, 3253 insertions(+) create mode 100644 routing/interval_budget_test.go create mode 100644 routing/interval_inflight_test.go create mode 100644 routing/interval_parallel_test.go create mode 100644 routing/interval_session.go create mode 100644 routing/interval_session_source.go create mode 100644 routing/interval_session_test.go diff --git a/routing/interval_budget_test.go b/routing/interval_budget_test.go new file mode 100644 index 00000000000..8a6684daf05 --- /dev/null +++ b/routing/interval_budget_test.go @@ -0,0 +1,497 @@ +package routing + +import ( + "math" + "testing" + + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +// The budget tests run over two corridors that differ in exactly two ways: one +// is free and unproven, the other charges a fee and has been watched carry the +// amount. Which one the session picks is then a pure statement about how it +// prices reliability against money. +const ( + // budgetCapacity is the capacity of every channel in the corridors. + budgetCapacity = btcutil.Amount(1_000_000) + + // budgetAmount is the amount paid, a tenth of a channel. + budgetAmount = lnwire.MilliSatoshi(100_000_000) + + // budgetHopFee is what the expensive corridor charges to forward. + budgetHopFee = lnwire.MilliSatoshi(200_000) +) + +// newBudgetSession builds a session over a free corridor through the first +// relay and a paying corridor through the second, and proves the paying one by +// recording that it has carried the amount. +// +// The fee limit is the one the payment is created with, which is what decides +// whether the session treats it as budgeted. Handing a limit to RequestRoute is +// not the same thing and deliberately does not classify. +func newBudgetSession(t *testing.T, feeLimit lnwire.MilliSatoshi) ( + *intervalPaymentSession, IntervalKey) { + + t.Helper() + + var ( + source = createPubkey(sourceNodeID) + cheap = createPubkey(firstRelayID) + dear = createPubkey(secondRelayID) + target = createPubkey(targetNodeID) + ) + + graph := ¶llelGraph{ + channels: []parallelChannel{ + { + id: 1, node1: source, node2: cheap, + capacity: budgetCapacity, + }, + { + id: 2, node1: cheap, node2: target, + capacity: budgetCapacity, + }, + { + id: 3, node1: source, node2: dear, + capacity: budgetCapacity, + }, + { + id: 4, node1: dear, node2: target, + capacity: budgetCapacity, baseFee: budgetHopFee, + }, + }, + } + + var paymentAddr [32]byte + payment := &LightningPayment{ + FinalCLTVDelta: 40, + FeeLimit: feeLimit, + Target: target, + PaymentAddr: fn.Some(paymentAddr), + Amount: budgetAmount, + CltvLimit: math.MaxUint32, + MaxParts: 1, + DestFeatures: lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + lnwire.PaymentAddrOptional, + lnwire.MPPOptional, + ), lnwire.Features, + ), + } + require.NoError(t, payment.SetPaymentHash([32]byte{})) + + getBandwidthHints := func(_ Graph) (bandwidthHints, error) { + return &mockBandwidthHints{ + hints: map[uint64]lnwire.MilliSatoshi{ + 1: lnwire.NewMSatFromSatoshis(budgetCapacity), + 3: lnwire.NewMSatFromSatoshis(budgetCapacity), + }, + }, nil + } + + store := NewIntervalStore(0) + + // The paying corridor's interior hop has been watched carry the amount, + // so it is near certain where the free corridor is only a guess. + proven := IntervalKey{ChanID: 4, From: dear, To: target} + store.RecordProbe( + proven, budgetAmount, + lnwire.NewMSatFromSatoshis(budgetCapacity), + ) + + session, err := newIntervalPaymentSession( + payment, source, getBandwidthHints, graph, store, + DefaultIntervalConfig(), + ) + require.NoError(t, err) + + return session, proven +} + +// TestIntervalBudgetPrice tests the exchange rate that decides whether a fee +// budget can influence the search at all. The units are the finding here: a +// rate proportional to the amount can never be reached by a realistic budget, +// while an absolute rate can. +func TestIntervalBudgetPrice(t *testing.T) { + t.Parallel() + + amt := lnwire.MilliSatoshi(1_000_000_000) + + // A payment with no budget has no rate at all. + require.Zero(t, newIntervalFeeRate(false, amt).price) + require.False(t, newIntervalFeeRate(false, amt).budgeted) + + // With a budget the rate is absolute and derived from what is left. + budget := lnwire.MilliSatoshi(400_000) + require.Equal( + t, float64(budget)/intervalBudgetShare, + intervalBudgetPrice(budget), + ) + + // The rate falls as the budget is spent, so a payment running low + // prices reliability ever more cheaply and stops paying up for it. + previous := math.MaxFloat64 + for _, left := range []lnwire.MilliSatoshi{ + 800_000, 400_000, 200_000, 120_000, + } { + current := intervalBudgetPrice(left) + require.Less(t, current, previous) + previous = current + } + + // The rate is bounded at both ends. A payment with almost nothing left + // still pays something, since a route it can afford beats no route. + require.Equal(t, intervalMinFeePrice, intervalBudgetPrice(1)) + require.Equal( + t, intervalMaxFeePrice, + intervalBudgetPrice(lnwire.MaxMilliSatoshi-1), + ) + + // Because the rate is absolute, its ceiling in relative terms tightens + // as the payment grows, which is the direction a budget quoted in parts + // per million needs. + rate := intervalBudgetPrice(budget) + require.Greater( + t, rate/float64(lnwire.MilliSatoshi(1_000_000)), + rate/float64(amt), + ) + + // Read as a price, the unbudgeted fallback is a fifth of the payment, + // which no fee budget anybody would set comes close to. That is the + // whole reason it never binds. + unbudgeted := intervalFeeRate{}.penalty(1, amt, intervalFeeWeight) + require.Less(t, unbudgeted, 1/(float64(amt)/10)) +} + +// TestIntervalFeePenaltyUnbudgetedIsVerbatim tests that the fee term of an +// unbudgeted payment is bit for bit the expression it has always been. +// +// This is a stricter test than it looks. The obvious refactor, precomputing +// the amount over the weight and dividing the fee by that, is algebraically +// the same and not the same in floating point, because it rounds twice where +// the original rounds once. The frontier compares these scores exactly, so a +// last-bit difference reorders labels and returns a different route. It was +// measured at 0.032 of objective on one tier. Equality here is therefore +// asserted on the bits, and the second half proves the assertion can fail. +func TestIntervalFeePenaltyUnbudgetedIsVerbatim(t *testing.T) { + t.Parallel() + + amounts := []lnwire.MilliSatoshi{ + 0, 1, 4, 5, 6, 1_000_000, 7_000_003, 33_333_337, + 100_000_000, 123_456_789, 200_000_000, + } + fees := []lnwire.MilliSatoshi{ + 0, 1, 3, 997, 100_000, 262_144, 499_999, 500_000, + } + weights := []float64{intervalFeeWeight, intervalShardFeeWeight} + + // reciprocal is the form this must never be written as. + reciprocal := func(fee float64, amt lnwire.MilliSatoshi, + weight float64) float64 { + + return fee / math.Max(float64(amt)/weight, 1) + } + + var differs int + for _, weight := range weights { + for _, amt := range amounts { + for _, feeAmt := range fees { + fee := float64(feeAmt) + + want := weight * fee / + math.Max(float64(amt), 1) + got := intervalFeeRate{}.penalty( + fee, amt, weight, + ) + + require.Equal(t, math.Float64bits(want), + math.Float64bits(got), + "amt=%v fee=%v weight=%v", amt, feeAmt, + weight) + + if reciprocal(fee, amt, weight) != want { + differs++ + } + } + } + } + + // The reciprocal form disagrees on a good fraction of these, which is + // what makes the equality above worth asserting rather than a ritual. + require.NotZero(t, differs, "the sweep found no case where the "+ + "reciprocal form differs, so it cannot catch the regression") + + // A payment with a budget takes the other branch and pays the rate the + // budget sets. + rate := newIntervalFeeRate(true, 400_000) + require.Equal( + t, 1_000/rate.price, + rate.penalty(1_000, 100_000_000, intervalFeeWeight), + ) +} + +// TestIntervalBudgetPicksCheapCorridor tests that a binding budget changes the +// route. With money effectively free the session buys the reliability it has +// evidence for, and with a budget tight enough that the same reliability costs +// more than it is worth, the session takes the cheap corridor instead. +func TestIntervalBudgetPicksCheapCorridor(t *testing.T) { + t.Parallel() + + var ( + cheap = createPubkey(firstRelayID) + dear = createPubkey(secondRelayID) + ) + + // With no budget the fee term is a rounding error against the risk of + // an unproven corridor, so the proven one wins. + session, _ := newBudgetSession(t, lnwire.MaxMilliSatoshi) + + rt, err := session.RequestRoute( + budgetAmount, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + require.Equal(t, dear, rt.Hops[0].PubKeyBytes) + require.EqualValues(t, budgetHopFee, rt.TotalAmount-budgetAmount) + + // Now hand the same session the same choice with a budget that can + // still afford the paying corridor twice over, but under which one nat + // of reliability is no longer worth what that corridor charges. + session, _ = newBudgetSession(t, budgetHopFee*2) + + rt, err = session.RequestRoute(budgetAmount, budgetHopFee*2, 0, 0, nil) + require.NoError(t, err) + require.Equal(t, cheap, rt.Hops[0].PubKeyBytes) + require.Zero(t, rt.TotalAmount-budgetAmount) +} + +// TestIntervalBudgetNeverExceeded tests the discipline lnd's own session has +// and this one must match: no route is ever returned that the payment cannot +// afford, at any budget. +func TestIntervalBudgetNeverExceeded(t *testing.T) { + t.Parallel() + + limits := []lnwire.MilliSatoshi{ + lnwire.MaxMilliSatoshi, budgetHopFee * 4, budgetHopFee * 2, + budgetHopFee, budgetHopFee - 1, budgetHopFee / 2, 1, + } + + for _, limit := range limits { + session, _ := newBudgetSession(t, limit) + + rt, err := session.RequestRoute(budgetAmount, limit, 0, 0, nil) + if err != nil { + require.ErrorIs(t, err, errNoPathFound) + + continue + } + + fee := rt.TotalAmount - budgetAmount + require.LessOrEqual(t, fee, limit, + "returned a route costing %v under a limit of %v", + fee, limit) + } + + // A budget too small for even the free corridor's zero fee is still + // routable, since the free corridor costs nothing. + session, _ := newBudgetSession(t, 0) + rt, err := session.RequestRoute(budgetAmount, 0, 0, 0, nil) + require.NoError(t, err) + require.Zero(t, rt.TotalAmount-budgetAmount) +} + +// TestIntervalFrontierKeepsCheapestLabel tests that the cheapest way out of a +// node is protected from eviction when the payment carries a fee budget, and +// only then. +// +// Under a budget the protection is what stops a frontier of reliable expensive +// labels from leaving a payment that cannot afford any of them with nothing. +// Without a budget it is a label kept for a limit that does not exist, +// displacing one that would have served the payment being made, and measurement +// found that costs real success on payments with no limit set. +func TestIntervalFrontierKeepsCheapestLabel(t *testing.T) { + t.Parallel() + + const deliver = lnwire.MilliSatoshi(1_000_000) + + // fill builds a frontier holding one cheap badly scoring label plus + // enough better scoring dearer ones to force eviction, and returns the + // cheap label and what the node ended up keeping. + fill := func(keepCheapest bool) (*intervalLabel, []*intervalLabel) { + node := route.Vertex{1} + frontier := &intervalFrontier{ + labels: map[route.Vertex][]*intervalLabel{}, + maxLabels: 3, + keepCheapest: keepCheapest, + } + + // The cheapest label is also the worst scoring one, so nothing + // but the protection would keep it. + cheapest := &intervalLabel{ + node: node, + netAmountReceived: deliver, + score: 100, + hops: 1, + } + require.True(t, frontier.insert(cheapest, deliver)) + + // Score falls as the amount rises across the rest of the set, so + // no label dominates another and each is a genuine trade-off the + // search would want to keep. + for i := 1; i <= 6; i++ { + frontier.insert(&intervalLabel{ + node: node, + netAmountReceived: deliver * + lnwire.MilliSatoshi(10+i), + score: float64(10 - i), + hops: 1, + }, deliver) + } + + kept := frontier.labels[node] + require.Len(t, kept, frontier.maxLabels) + + return cheapest, kept + } + + // With a budget the cheap label survives, and it is still the cheapest + // thing the node holds. + cheapest, kept := fill(true) + + require.Contains(t, kept, cheapest) + require.True(t, cheapest.active) + + for _, label := range kept { + require.GreaterOrEqual( + t, label.netAmountReceived, + cheapest.netAmountReceived, + ) + } + + // With no budget it is evicted on its score like any other label, which + // is the behaviour the search had before fee budgets were priced at all. + cheapest, kept = fill(false) + + require.NotContains(t, kept, cheapest) + require.False(t, cheapest.active) + + for _, label := range kept { + require.Less(t, label.score, cheapest.score) + } +} + +// TestIntervalBudgeted tests the switch that decides both how fees are priced +// and which of the two eviction rules a search uses. Anything short of the +// sentinel is a real limit a route can exceed, so anything short of it counts. +func TestIntervalBudgeted(t *testing.T) { + t.Parallel() + + require.False(t, intervalBudgeted(lnwire.MaxMilliSatoshi)) + + for _, limit := range []lnwire.MilliSatoshi{ + 0, 1, budgetHopFee, lnwire.MaxMilliSatoshi - 1, + } { + require.True(t, intervalBudgeted(limit), + "a limit of %v should price fees", limit) + } +} + +// TestIntervalBudgetSurvivesShards tests the bug this classification exists to +// avoid, and its mirror. +// +// RequestRoute is handed the budget remaining, not the budget. An unbudgeted +// payment therefore carries the no-limit sentinel only until its first shard +// pays a fee; from the second shard on it carries the sentinel minus that fee, +// which is an ordinary looking number. Reading that as "this payment has a +// budget" flipped every unbudgeted payment that splits onto the budgeted +// branch, priced its fees against a limit nobody set, and turned on a frontier +// protection it had no use for. +func TestIntervalBudgetSurvivesShards(t *testing.T) { + t.Parallel() + + // What the lifecycle hands the second route request of a payment whose + // first shard paid a fee. It is not the sentinel, and that is the point. + const feesPaid = budgetHopFee + + afterFirstShard := lnwire.MaxMilliSatoshi - feesPaid + require.NotEqual(t, lnwire.MaxMilliSatoshi, afterFirstShard) + + // A payment created with no limit stays unbudgeted across its shards, + // however much of the sentinel its shards have eaten. + session, _ := newBudgetSession(t, lnwire.MaxMilliSatoshi) + require.False(t, session.budgeted) + + for _, remaining := range []lnwire.MilliSatoshi{ + lnwire.MaxMilliSatoshi, afterFirstShard, + lnwire.MaxMilliSatoshi - feesPaid*2, budgetHopFee, + } { + rate := session.feeRate(remaining) + + // No budget branch, so no budget price, no cheapest-label keep, + // and a fee term that is the verbatim amount relative + // expression. + require.False(t, rate.budgeted, + "reclassified as budgeted at a remainder of %v", + remaining) + require.Zero(t, rate.price) + require.Equal( + t, intervalFeeWeight*float64(budgetHopFee)/ + math.Max(float64(budgetAmount), 1), + rate.penalty( + float64(budgetHopFee), budgetAmount, + intervalFeeWeight, + ), + ) + } + + // Driving two requests through the session leaves the latch alone, which + // is the whole of the fix. + _, err := session.RequestRoute( + budgetAmount, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + require.False(t, session.budgeted) + + _, err = session.RequestRoute(budgetAmount, afterFirstShard, 0, 0, nil) + require.NoError(t, err) + require.False(t, session.budgeted) + require.False(t, session.feeRate(afterFirstShard).budgeted) + + // The mirror: a payment created with a limit stays budgeted across its + // shards, and the rate it pays follows the shrinking remainder down. + budgeted, _ := newBudgetSession(t, budgetHopFee*4) + require.True(t, budgeted.budgeted) + + // These remainders are all under the rate ceiling, so the rate they + // produce actually moves rather than pinning to the clamp. + previous := math.MaxFloat64 + for _, remaining := range []lnwire.MilliSatoshi{ + budgetHopFee * 4, budgetHopFee * 3, budgetHopFee * 2, + } { + rate := budgeted.feeRate(remaining) + + require.True(t, rate.budgeted) + require.Equal(t, intervalBudgetPrice(remaining), rate.price) + require.Less(t, rate.price, previous) + previous = rate.price + + // The budgeted branch prices off the rate rather than off the + // amount. + require.Equal( + t, float64(budgetHopFee)/rate.price, + rate.penalty( + float64(budgetHopFee), budgetAmount, + intervalFeeWeight, + ), + ) + } + + _, err = budgeted.RequestRoute(budgetAmount, budgetHopFee*4, 0, 0, nil) + require.NoError(t, err) + require.True(t, budgeted.budgeted) + require.True(t, budgeted.feeRate(budgetHopFee*2).budgeted) +} diff --git a/routing/interval_inflight_test.go b/routing/interval_inflight_test.go new file mode 100644 index 00000000000..c4c2d6f467a --- /dev/null +++ b/routing/interval_inflight_test.go @@ -0,0 +1,384 @@ +package routing + +import ( + "math" + "testing" + + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +// Node ids for the in-flight tests, on top of the ones the other interval +// tests already define. +const ( + thirdRelayID = 5 + fourthRelayID = 6 +) + +// newCorridorSession builds a session over a graph offering two disjoint two +// hop corridors from us to the target, and returns the session and the node +// wide store behind it. +// +// The corridors are deliberately identical in capacity and policy, so that +// nothing but liquidity beliefs can separate them. +func newCorridorSession(t *testing.T, amt lnwire.MilliSatoshi, + maxParts uint32) (*intervalPaymentSession, *IntervalStore) { + + t.Helper() + + const capacity = btcutil.Amount(1_000_000) + + var ( + source = createPubkey(sourceNodeID) + first = createPubkey(firstRelayID) + second = createPubkey(secondRelayID) + target = createPubkey(targetNodeID) + ) + + graph := ¶llelGraph{ + channels: []parallelChannel{ + {id: 1, node1: source, node2: first, capacity: capacity}, + {id: 2, node1: first, node2: target, capacity: capacity}, + {id: 3, node1: source, node2: second, capacity: capacity}, + {id: 4, node1: second, node2: target, capacity: capacity}, + }, + } + + var paymentAddr [32]byte + payment := &LightningPayment{ + FinalCLTVDelta: 40, + FeeLimit: lnwire.MaxMilliSatoshi, + Target: target, + PaymentAddr: fn.Some(paymentAddr), + Amount: amt, + CltvLimit: math.MaxUint32, + MaxParts: maxParts, + DestFeatures: lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + lnwire.PaymentAddrOptional, + lnwire.MPPOptional, + ), lnwire.Features, + ), + } + require.NoError(t, payment.SetPaymentHash([32]byte{})) + + getBandwidthHints := func(_ Graph) (bandwidthHints, error) { + return &mockBandwidthHints{ + hints: map[uint64]lnwire.MilliSatoshi{ + 1: lnwire.NewMSatFromSatoshis(capacity), + 3: lnwire.NewMSatFromSatoshis(capacity), + }, + }, nil + } + + store := NewIntervalStore(0) + + cfg := DefaultIntervalConfig() + cfg.MinShardAmt = lnwire.NewMSatFromSatoshis(1_000) + + session, err := newIntervalPaymentSession( + payment, source, getBandwidthHints, graph, store, cfg, + ) + require.NoError(t, err) + + return session, store +} + +// relayOf returns the relay a two hop route went through. +func relayOf(t *testing.T, rt *route.Route) route.Vertex { + t.Helper() + + require.Len(t, rt.Hops, 2) + + return rt.Hops[0].PubKeyBytes +} + +// TestIntervalInFlightPricesOwnHolds tests that a second shard prices an +// interior corridor knowing what the first shard is already holding on it. +// +// Nothing in the graph distinguishes the two corridors, and the sender's own +// channels are covered by the bandwidth hints, so the only way the second shard +// can prefer the untouched corridor is by counting the HTLC it already has in +// flight on the other one. +func TestIntervalInFlightPricesOwnHolds(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + session, store := newCorridorSession(t, amt, 4) + + // The first shard takes one of the two corridors. + first, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + firstRelay := relayOf(t, first) + + // It is now holding the interior hop of that corridor, and the store + // says so to anyone who asks. + interior := IntervalKey{ + ChanID: intervalChanIDOf(first, 1), + From: firstRelay, + To: createPubkey(targetNodeID), + } + require.Equal(t, first.Hops[0].AmtToForward, store.Held(interior)) + + // The second shard, asked for while the first is still in flight, takes + // the other corridor. + second, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 1, 0, nil) + require.NoError(t, err) + require.NotEqual(t, firstRelay, relayOf(t, second)) + + // The preference is the hold and nothing else: with the hold released, + // the search is free to return the first corridor again. + session.ReportAttemptFailure( + 0, first, nil, lnwire.NewTemporaryChannelFailure(nil), + ) + require.Zero(t, store.Held(interior)) +} + +// TestIntervalInFlightRaisesEffectiveAmount tests the arithmetic underneath the +// preference above. A hop holding H of ours must have had A plus H available +// when we last looked at it, so a hold can rule a shard out on its own. +func TestIntervalInFlightRaisesEffectiveAmount(t *testing.T) { + t.Parallel() + + capacity := lnwire.NewMSatFromSatoshis(1_000_000) + amt := capacity / 2 + + session, store := newCorridorSession(t, amt, 4) + + key := IntervalKey{ + ChanID: 2, + From: createPubkey(firstRelayID), + To: createPubkey(targetNodeID), + } + + // With nothing held, the hop is priced on the prior alone. + cold := session.edgeProbability(key, amt, capacity) + require.Greater(t, cold, 0.0) + + // We have watched the hop carry the whole amount, so it is near + // certain. + store.RecordProbe(key, amt, capacity) + require.EqualValues( + t, intervalProvenProbability, + session.edgeProbability(key, amt, capacity), + ) + + // Now hold half the amount on it. The hop is no longer being asked for + // something we have proven, it is being asked for half again as much, + // and the price drops accordingly. + store.Hold(map[IntervalKey]lnwire.MilliSatoshi{key: amt / 2}) + + withHold := session.edgeProbability(key, amt, capacity) + require.Less(t, withHold, intervalProvenProbability) + + // A hold that takes the pair past what we have watched fail rules the + // shard out entirely, without the router having to spend an attempt to + // find that out. + store.RecordFailure(key, amt+amt/2, capacity) + require.Zero(t, session.edgeProbability(key, amt, capacity)) +} + +// TestIntervalInFlightIgnoresFirstHop tests that we do not charge our own +// channels twice. The switch already nets in-flight HTLCs out of the bandwidth +// it reports for our links, so the pathfinder has heard about them once +// already. +func TestIntervalInFlightIgnoresFirstHop(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + session, store := newCorridorSession(t, amt, 4) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + + // The interior hop is held, the hop out of our own node is not. + own := IntervalKey{ + ChanID: rt.Hops[0].ChannelID, + From: createPubkey(sourceNodeID), + To: relayOf(t, rt), + } + require.Zero(t, store.Held(own)) + require.NotZero(t, store.Held(IntervalKey{ + ChanID: intervalChanIDOf(rt, 1), + From: relayOf(t, rt), + To: createPubkey(targetNodeID), + })) +} + +// TestIntervalInFlightReleaseOnSettle tests that a settled shard gives its hold +// back, and that the settlement itself is what carries the liquidity out of the +// picture from then on. +func TestIntervalInFlightReleaseOnSettle(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + session, store := newCorridorSession(t, amt, 4) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + require.NotZero(t, store.HeldLen()) + + session.ReportAttemptSuccess(0, rt) + + require.Zero(t, store.HeldLen()) + require.Empty(t, session.outstanding) +} + +// TestIntervalInFlightReleaseOnFailure tests the same for a shard that failed. +func TestIntervalInFlightReleaseOnFailure(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + session, store := newCorridorSession(t, amt, 4) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + require.NotZero(t, store.HeldLen()) + + failIndex := 1 + session.ReportAttemptFailure( + 0, rt, &failIndex, lnwire.NewTemporaryChannelFailure(nil), + ) + + require.Zero(t, store.HeldLen()) + require.Empty(t, session.outstanding) +} + +// TestIntervalInFlightReleaseOnTeardown tests the case nothing else covers: a +// route the session handed out that never became an HTLC, so no outcome is ever +// reported for it. Without the teardown sweep its hold would sit on the node +// wide store for as long as the process lived, depressing a channel with +// nothing behind it. +func TestIntervalInFlightReleaseOnTeardown(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + session, store := newCorridorSession(t, amt, 4) + + _, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + require.NotZero(t, store.HeldLen()) + + // The lifecycle exits without the route ever reaching the switch. + session.ReleaseAttempts() + + require.Zero(t, store.HeldLen()) + require.Empty(t, session.outstanding) + + // Releasing again is harmless, since the lifecycle may well have + // reported an outcome first. + session.ReleaseAttempts() + require.Zero(t, store.HeldLen()) +} + +// TestIntervalInFlightReconcilesAgainstLifecycle tests the net that catches a +// hold while a payment is still running. The number of HTLCs in flight comes +// from the payments database, so when we are holding more shards than that, the +// extra ones never made it out and are dropped. +func TestIntervalInFlightReconcilesAgainstLifecycle(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(400_000) + session, store := newCorridorSession(t, amt, 8) + + // Two routes handed out, neither reported. + _, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + _, err = session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 1, 0, nil) + require.NoError(t, err) + require.Len(t, session.outstanding, 2) + + // The lifecycle now says only one HTLC is actually in flight, so the + // older hold is given back. + _, err = session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 1, 0, nil) + require.NoError(t, err) + + // One dropped by the reconciliation, one added by the request above. + require.Len(t, session.outstanding, 2) + + // And with the payment reporting nothing in flight at all, everything + // is given back. + session.reconcileHolds(0) + require.Empty(t, session.outstanding) + require.Zero(t, store.HeldLen()) +} + +// TestIntervalInFlightHoldsAtPairScope tests that a hold lands under the same +// key the pricing uses. On a pair with several channels an observation cannot +// name one of them, so neither can a hold: it belongs to the pair. +func TestIntervalInFlightHoldsAtPairScope(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(100_000) + session, store := newParallelSession(t, 2, amt) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + + var ( + relay = createPubkey(firstRelayID) + target = createPubkey(targetNodeID) + ) + + // Neither channel of the pair carries the hold on its own. + for _, chanID := range []uint64{100, 101} { + require.Zero(t, store.Held(IntervalKey{ + ChanID: chanID, From: relay, To: target, + })) + } + + // The pair carries it, which is the key the search prices under. + pair := IntervalKey{From: relay, To: target} + require.True(t, pair.IsPairScoped()) + require.Equal(t, rt.Hops[0].AmtToForward, store.Held(pair)) + + // And it is given back at the same scope. + session.ReportAttemptSuccess(0, rt) + require.Zero(t, store.Held(pair)) + require.Zero(t, store.HeldLen()) +} + +// TestIntervalInFlightSharedAcrossPayments tests that the overlay is node wide. +// A second payment, with its own session, prices a corridor knowing what the +// first payment is holding on it. +func TestIntervalInFlightSharedAcrossPayments(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(600_000) + first, store := newCorridorSession(t, amt, 4) + + // A second payment sharing the same node wide store. + second, _ := newCorridorSession(t, amt, 4) + second.store = store + + firstRoute, err := first.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + + // The other payment, which has sent nothing itself, still steps around + // the corridor the first one is using. + secondRoute, err := second.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + require.NotEqual( + t, relayOf(t, firstRoute), relayOf(t, secondRoute), + ) + + // Each payment gives back only its own hold. + first.ReleaseAttempts() + require.Equal(t, 1, store.HeldLen()) + + second.ReleaseAttempts() + require.Zero(t, store.HeldLen()) +} + +// intervalChanIDOf returns the channel id of the given hop of a route. +func intervalChanIDOf(rt *route.Route, hop int) uint64 { + return rt.Hops[hop].ChannelID +} diff --git a/routing/interval_parallel_test.go b/routing/interval_parallel_test.go new file mode 100644 index 00000000000..b2382ce36e5 --- /dev/null +++ b/routing/interval_parallel_test.go @@ -0,0 +1,290 @@ +package routing + +import ( + "context" + "math" + "testing" + + "github.com/btcsuite/btcd/btcutil/v2" + graphdb "github.com/lightningnetwork/lnd/graph/db" + "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +// parallelChannel is one channel of the parallel graph below. +type parallelChannel struct { + id uint64 + node1 route.Vertex + node2 route.Vertex + capacity btcutil.Amount + + // baseFee is what the channel charges to forward, which the fee + // budget tests need and the rest leave at zero. + baseFee lnwire.MilliSatoshi +} + +// parallelGraph is a channel graph that, unlike the mock graph the other tests +// use, can hold more than one channel between the same pair of nodes. That is +// the whole point of it: the attribution question this file tests only arises +// when a peer has a choice of channels to forward over. +type parallelGraph struct { + channels []parallelChannel +} + +// A compile time assertion that the graph satisfies both interfaces the +// interval session needs. +var _ Graph = (*parallelGraph)(nil) +var _ GraphSessionFactory = (*parallelGraph)(nil) + +// ForEachNodeDirectedChannel calls the callback for every channel of the given +// node. +// +// NOTE: Part of the Graph interface. +func (g *parallelGraph) ForEachNodeDirectedChannel(_ context.Context, + nodePub route.Vertex, cb func(*graphdb.DirectedChannel) error, + _ func()) error { + + for _, channel := range g.channels { + var other route.Vertex + switch nodePub { + case channel.node1: + other = channel.node2 + + case channel.node2: + other = channel.node1 + + default: + continue + } + + toNode := nodePub + err := cb(&graphdb.DirectedChannel{ + ChannelID: channel.id, + OtherNode: other, + Capacity: channel.capacity, + OutPolicySet: true, + InPolicy: &models.CachedEdgePolicy{ + ChannelID: channel.id, + ToNodePubKey: func() route.Vertex { + return toNode + }, + ToNodeFeatures: lnwire.EmptyFeatureVector(), + FeeBaseMSat: channel.baseFee, + }, + }) + if err != nil { + return err + } + } + + return nil +} + +// FetchNodeFeatures returns the features of the given node. +// +// NOTE: Part of the Graph interface. +func (g *parallelGraph) FetchNodeFeatures(_ context.Context, + _ route.Vertex) (*lnwire.FeatureVector, error) { + + return lnwire.EmptyFeatureVector(), nil +} + +// GraphSession hands the callback access to the graph. +// +// NOTE: Part of the GraphSessionFactory interface. +func (g *parallelGraph) GraphSession(_ context.Context, + cb func(graph graphdb.NodeTraverser) error, _ func()) error { + + return cb(g) +} + +// newParallelSession builds a session over a graph in which the relay reaches +// the target over the given number of channels. +func newParallelSession(t *testing.T, siblings int, amt lnwire.MilliSatoshi) ( + *intervalPaymentSession, *IntervalStore) { + + t.Helper() + + const capacity = btcutil.Amount(1_000_000) + + var ( + source = createPubkey(sourceNodeID) + relay = createPubkey(firstRelayID) + target = createPubkey(targetNodeID) + ) + + // One channel from us to the relay, then the requested number from the + // relay onwards to the target. + graph := ¶llelGraph{ + channels: []parallelChannel{ + {id: 1, node1: source, node2: relay, capacity: capacity}, + }, + } + for i := 0; i < siblings; i++ { + graph.channels = append(graph.channels, parallelChannel{ + id: uint64(100 + i), + node1: relay, + node2: target, + capacity: capacity, + }) + } + + payment := &LightningPayment{ + FinalCLTVDelta: 40, + FeeLimit: lnwire.MaxMilliSatoshi, + Target: target, + Amount: amt, + CltvLimit: math.MaxUint32, + MaxParts: 1, + DestFeatures: lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + ), lnwire.Features, + ), + } + require.NoError(t, payment.SetPaymentHash([32]byte{})) + + getBandwidthHints := func(_ Graph) (bandwidthHints, error) { + return &mockBandwidthHints{ + hints: map[uint64]lnwire.MilliSatoshi{ + 1: lnwire.NewMSatFromSatoshis(capacity), + }, + }, nil + } + + store := NewIntervalStore(0) + session, err := newIntervalPaymentSession( + payment, source, getBandwidthHints, graph, store, + DefaultIntervalConfig(), + ) + require.NoError(t, err) + + return session, store +} + +// TestIntervalParallelChannelAttribution tests the invariant that guards this +// model against non-strict forwarding. A node asked to forward over one channel +// may use any channel it has to the same peer, and the onion failure that comes +// back names neither. So when a pair has several channels, a failure must never +// leave a hard bound on one of them, because the bound would be a claim the +// evidence cannot support and this model has no way to take it back. +func TestIntervalParallelChannelAttribution(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(100_000) + session, store := newParallelSession(t, 2, amt) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + require.Len(t, rt.Hops, 2) + + // The relay refuses to forward onwards. + failIndex := 1 + session.ReportAttemptFailure( + 0, rt, &failIndex, lnwire.NewTemporaryChannelFailure(nil), + ) + + var ( + relay = createPubkey(firstRelayID) + target = createPubkey(targetNodeID) + capacity = lnwire.NewMSatFromSatoshis(1_000_000) + ) + + // Neither channel of the pair carries a bound, because the failure + // cannot say which of them was tried. + for _, chanID := range []uint64{100, 101} { + key := IntervalKey{ChanID: chanID, From: relay, To: target} + + require.Zero(t, store.Get(key, capacity).UpperFail, + "channel %v was blamed for a failure that could not "+ + "name it", chanID) + require.NotZero( + t, store.Probability(key, amt, capacity), + "channel %v was ruled out by a failure that could "+ + "not name it", chanID) + } + + // The pair carries it instead, which is exactly what was observed: + // this peer could not move this amount to that node. + pair := IntervalKey{From: relay, To: target} + require.True(t, pair.IsPairScoped()) + + interval := store.Get(pair, capacity) + require.True(t, interval.Known) + require.NotZero(t, interval.UpperFail) + require.LessOrEqual(t, interval.UpperFail, rt.Hops[0].AmtToForward) + + // The bound is applied on the next search, so the pair is no longer + // offered the amount it just refused. + require.Zero(t, store.Probability(pair, amt, capacity)) + + _, err = session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.ErrorIs(t, err, errNoPathFound) +} + +// TestIntervalSingleChannelKeepsChannelScope tests that the fallback above only +// fires when it has to. A pair with one channel between it is unambiguous, and +// keeping channel scope there is the whole reason this model can hold an +// interval that means something physical. +func TestIntervalSingleChannelKeepsChannelScope(t *testing.T) { + t.Parallel() + + amt := lnwire.NewMSatFromSatoshis(100_000) + session, store := newParallelSession(t, 1, amt) + + rt, err := session.RequestRoute(amt, lnwire.MaxMilliSatoshi, 0, 0, nil) + require.NoError(t, err) + + failIndex := 1 + session.ReportAttemptFailure( + 0, rt, &failIndex, lnwire.NewTemporaryChannelFailure(nil), + ) + + var ( + relay = createPubkey(firstRelayID) + target = createPubkey(targetNodeID) + capacity = lnwire.NewMSatFromSatoshis(1_000_000) + ) + + // The bound landed on the channel itself. + key := IntervalKey{ChanID: 100, From: relay, To: target} + require.NotZero(t, store.Get(key, capacity).UpperFail) + + // Nothing was written about the pair, since there was no ambiguity to + // resolve. + require.False( + t, store.Get( + IntervalKey{From: relay, To: target}, capacity, + ).Known, + ) +} + +// TestIntervalScopeKey tests the rule itself. +func TestIntervalScopeKey(t *testing.T) { + t.Parallel() + + key := IntervalKey{ + ChanID: 7, + From: route.Vertex{1}, + To: route.Vertex{2}, + } + + // A pair the search never looked at, and a pair with exactly one + // channel, are both named by their channel. + require.Equal(t, key, intervalScopeKey(key, 0)) + require.Equal(t, key, intervalScopeKey(key, 1)) + require.False(t, key.IsPairScoped()) + + // More than one channel means the observation cannot name one. + scoped := intervalScopeKey(key, 2) + require.True(t, scoped.IsPairScoped()) + require.Equal(t, key.From, scoped.From) + require.Equal(t, key.To, scoped.To) + + // A pair scoped key still has two directions, since the liquidity it + // describes still sits on one side or the other. + require.Equal(t, key.To, scoped.Reverse().From) + require.True(t, scoped.Reverse().IsPairScoped()) +} diff --git a/routing/interval_session.go b/routing/interval_session.go new file mode 100644 index 00000000000..e689794ff21 --- /dev/null +++ b/routing/interval_session.go @@ -0,0 +1,1327 @@ +package routing + +import ( + "context" + "errors" + "fmt" + "math" + "strings" + "sync" + + "github.com/btcsuite/btclog/v2" + graphdb "github.com/lightningnetwork/lnd/graph/db" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" +) + +// The weights below price a candidate shard once a route has been found for it. +// The stock session never makes this comparison, because it does not choose a +// shard size: it asks for the whole remaining amount and halves it whenever no +// route comes back. This session prices every rung of a ladder and takes the +// best, which is what lets it split before it has failed at all. +const ( + // intervalShardFeeWeight sets the fee sensitivity of the shard score + // when the payment carries no fee budget, in the same units and for the + // same reasons as intervalFeeWeight. See intervalFeePenalty. + intervalShardFeeWeight = 4.0 + + // intervalShardHopWeight prices each hop of the shard's route, which + // breaks ties towards the shorter of two otherwise equal routes. + intervalShardHopWeight = 0.006 + + // intervalCompletionBonus is added to a shard that covers the whole + // remaining amount. Finishing a payment in one HTLC is worth a little + // more than the sum of its parts suggests, because every extra shard is + // another chance to fail and another HTLC to hold open. + intervalCompletionBonus = 0.08 + + // The split appetite weights below decide how much a larger shard is + // worth relative to its risk, and they respond to how the payment is + // going. Once a part has settled we are committed and want to finish; + // after several failures we would rather cut smaller and make progress. + intervalAppetiteDefault = 0.72 + intervalAppetiteCommitted = 0.94 + intervalAppetiteStruggling = 0.50 + intervalAppetiteCautious = 0.60 + + // intervalStrugglingAfter is the number of failed attempts after which + // the session turns cautious about shard size. + intervalStrugglingAfter = 3 +) + +// Session penalties. These are per payment and die with it, unlike the interval +// store, which outlives every payment. A penalty here says "this payment has a +// reason to avoid this channel", not "this channel is bad". +const ( + // intervalFailurePenalty is applied to a channel this payment watched + // fail. + intervalFailurePenalty = 1.35 + + // intervalPolicyPenalty is applied to a channel that rejected the HTLC + // on a policy grounds, such as an insufficient fee or a bad expiry. + // Such a channel is also blocked for the rest of the payment, since the + // policy we hold for it is stale and a retry would be built from the + // same stale policy. + intervalPolicyPenalty = 6 + + // intervalUnknownPenalty is applied to a channel that failed for a + // reason we have no model for. + intervalUnknownPenalty = 3 + + // intervalContradictionPenalty is applied across a whole route when an + // unattributable failure names no suspect at all, which means something + // we believe is wrong but we cannot tell what. + intervalContradictionPenalty = 0.35 + + // intervalRoutePenalty is applied across a whole route on a failure + // that carries no liquidity information. + intervalRoutePenalty = 1.0 + + // intervalSuspicionMass is spread across the suspects of an + // unattributable failure, divided by the square root of how many there + // are, so that a failure with one plausible cause speaks far louder + // than one with ten. + intervalSuspicionMass = 2.2 + + // intervalSuspicionEscalation and intervalSuspicionCertainty are the + // counts at which a channel that keeps turning up in unexplained + // failures earns an extra penalty and then a hard bound. This is a + // Bayesian argument in the shape of a counter. + intervalSuspicionEscalation = 4 + intervalSuspicionCertainty = 8 + + // intervalExtraSuspicionPenalty is the extra penalty applied at the + // escalation threshold. + intervalExtraSuspicionPenalty = 0.30 + + // intervalPenaltyWeight converts an accumulated penalty into a + // probability multiplier, and intervalPenaltyCap bounds how much + // penalty can pile onto one channel. + intervalPenaltyWeight = 0.70 + intervalPenaltyCap = 8 + + // intervalSettleDecay and intervalProbeDecay are the factors a penalty + // is multiplied by when the channel proves itself, by settling a part + // or by forwarding one. A channel that works is forgiven quickly, but + // not instantly. + intervalSettleDecay = 0.12 + intervalProbeDecay = 0.25 + + // intervalPenaltyFloor is the level below which a decayed penalty is + // dropped rather than kept around at a value that no longer matters. + intervalPenaltyFloor = 0.25 +) + +// intervalShardDivisors are the divisors applied to an amount this payment has +// already proven does not fit, to produce shard sizes that sit just under it. +// This is what makes the ladder a function of the beliefs and not only of the +// amount. +var intervalShardDivisors = []lnwire.MilliSatoshi{2, 4, 8, 16, 32} + +// intervalShardMultiples are small multiples of the smallest shard that would +// still let the payment finish within its part budget. +var intervalShardMultiples = []lnwire.MilliSatoshi{2, 3, 4, 6, 8} + +// intervalPaymentSession is a PaymentSession that owns route selection and MPP +// splitting together. Where the stock session asks path finding for the whole +// remaining amount and halves it when nothing comes back, this one enumerates a +// ladder of candidate shard sizes, finds a route for each, and picks the best +// pairing of the two. Both halves of that decision read the same liquidity +// intervals, so a failure at one amount reshapes not just which route is tried +// next but which amounts are considered at all. +type intervalPaymentSession struct { + additionalEdges + + selfNode route.Vertex + + payment *LightningPayment + + getBandwidthHints func(Graph) (bandwidthHints, error) + + graphSessFactory GraphSessionFactory + + // store is the node wide belief about channel liquidity, shared with + // every other payment. + store *IntervalStore + + cfg IntervalConfig + + log btclog.Logger + + // mu guards everything below it. RequestRoute and the result reporting + // methods are all driven from the payment lifecycle's own goroutine, + // but the session's contract says it is safe for concurrent access, so + // we make it so. + mu sync.Mutex + + // penalties, blocked, failedAt and suspects are this payment's own view + // of the channels it has touched, and they die with the payment. + penalties map[IntervalKey]float64 + blocked map[IntervalKey]struct{} + failedAt map[IntervalKey]lnwire.MilliSatoshi + suspects map[IntervalKey]uint32 + + // routeFailedAt remembers the smallest amount at which a given route + // has failed, so that the same route is not handed out again at an + // amount we already know it cannot carry. + routeFailedAt map[string]lnwire.MilliSatoshi + + // capacities remembers the capacity we path found each directed channel + // against, so that an outcome reported later can be recorded against + // the same scale it was priced with. + capacities map[IntervalKey]lnwire.MilliSatoshi + + // scopes remembers, for each hop of a route we dispatched, the key the + // hop was priced under. A hop across a pair with several channels is + // priced and recorded at pair scope, because nothing in an onion + // failure says which of the channels carried the payment. + scopes map[IntervalKey]IntervalKey + + // outstanding holds the shards this session has handed to the payment + // lifecycle and not yet seen resolved, oldest first. Every entry is + // mirrored into the node wide overlay, so this slice is also the record + // of what this session owes back to it. + outstanding []*heldShard + + // budgeted is latched at construction from the fee limit this payment + // was created with. It decides which way fees are priced and whether the + // search protects the cheapest label, and it must never be re-derived + // from the remaining budget the lifecycle hands RequestRoute. + budgeted bool + + attempts uint32 + failedAttempts uint32 + settledParts uint32 +} + +// heldShard is one route this session returned, and the amounts it committed +// on each directed channel of that route. +type heldShard struct { + // routeKey identifies the route, so that an outcome reported later can + // be matched back to the shard that produced it. + routeKey string + + // amounts is what the shard committed, keyed at the same scope the + // route was priced under. + amounts map[IntervalKey]lnwire.MilliSatoshi +} + +// A compile time assertion to ensure the interval session satisfies both the +// session contract and the optional result reporting one. +var _ PaymentSession = (*intervalPaymentSession)(nil) +var _ PaymentResultReporter = (*intervalPaymentSession)(nil) + +// newIntervalPaymentSession builds a session for one payment. +func newIntervalPaymentSession(p *LightningPayment, selfNode route.Vertex, + getBandwidthHints func(Graph) (bandwidthHints, error), + graphSessFactory GraphSessionFactory, store *IntervalStore, + cfg IntervalConfig) (*intervalPaymentSession, error) { + + edges, err := RouteHintsToEdges(p.RouteHints, p.Target) + if err != nil { + return nil, err + } + + cfg.fillDefaults() + + logPrefix := fmt.Sprintf("IntervalSession(%x):", p.Identifier()) + + return &intervalPaymentSession{ + // Whether this payment has a budget is settled here and never + // asked again. RequestRoute is handed the budget remaining + // rather than the budget, and a limit with fees subtracted from + // it stops looking like the no-limit sentinel after the first + // shard pays anything, so a payment with no budget that splits + // would otherwise reclassify itself as having one. + budgeted: intervalBudgeted(p.FeeLimit), + additionalEdges: edges, + selfNode: selfNode, + payment: p, + getBandwidthHints: getBandwidthHints, + graphSessFactory: graphSessFactory, + store: store, + cfg: cfg, + log: log.WithPrefix(logPrefix), + penalties: make(map[IntervalKey]float64), + blocked: make(map[IntervalKey]struct{}), + failedAt: make(map[IntervalKey]lnwire.MilliSatoshi), + suspects: make(map[IntervalKey]uint32), + routeFailedAt: make(map[string]lnwire.MilliSatoshi), + capacities: make(map[IntervalKey]lnwire.MilliSatoshi), + scopes: make(map[IntervalKey]IntervalKey), + }, nil +} + +// intervalChoice is one candidate pairing of a shard size with the route found +// for it. +type intervalChoice struct { + route *route.Route + edges []*unifiedEdge + cache *intervalGraphCache + shard lnwire.MilliSatoshi + utility float64 +} + +// RequestRoute returns the next route to attempt, which may carry the whole +// remaining amount or only a shard of it. +// +// NOTE: This function is safe for concurrent access. +// NOTE: Part of the PaymentSession interface. +func (p *intervalPaymentSession) RequestRoute(maxAmt, + feeLimit lnwire.MilliSatoshi, activeShards, height uint32, + firstHopCustomRecords lnwire.CustomRecords) (*route.Route, error) { + + p.mu.Lock() + defer p.mu.Unlock() + + if maxAmt == 0 { + return nil, errNoPathFound + } + + // The lifecycle reads the number of HTLCs in flight from the payments + // database, so it is the one count we can trust. Reconcile our own + // record of what we are holding against it before pricing anything. + p.reconcileHolds(activeShards) + + // A session that believes it can always find one more route would + // otherwise spin until the payment times out. + if p.attempts >= p.cfg.AttemptLimit { + p.log.Debugf("Giving up after %v attempts", p.attempts) + + return nil, errNoPathFound + } + + // Respect the client side maximum shard size if one is set. + if p.payment.MaxShardAmt != nil && maxAmt > *p.payment.MaxShardAmt { + p.log.Debugf("Clamping payment attempt from %v to %v due to "+ + "max shard size of %v", maxAmt, *p.payment.MaxShardAmt, + *p.payment.MaxShardAmt) + + maxAmt = *p.payment.MaxShardAmt + } + + // Add BlockPadding to the finalCltvDelta so that the receiving node + // does not reject the HTLC if some blocks are mined while it's in + // flight. + finalCltvDelta := p.payment.FinalCLTVDelta + BlockPadding + + // The final delta is subtracted before path finding, because the + // optimal path does not depend on it. + restrictions := &RestrictParams{ + FeeLimit: feeLimit, + OutgoingChannelIDs: p.payment.OutgoingChannelIDs, + LastHop: p.payment.LastHop, + CltvLimit: p.payment.CltvLimit - uint32(finalCltvDelta), + DestCustomRecords: p.payment.DestCustomRecords, + DestFeatures: p.payment.DestFeatures, + PaymentAddr: p.payment.PaymentAddr, + Amp: p.payment.amp, + Metadata: p.payment.Metadata, + FirstHopCustomRecords: firstHopCustomRecords, + } + + finalHtlcExpiry := int32(height) + int32(finalCltvDelta) + + partsLeft := p.partsLeft(activeShards) + if partsLeft == 0 { + p.log.Debugf("Not requesting a route, the part limit of %v "+ + "has been reached", p.payment.MaxParts) + + return nil, errNoPathFound + } + + // The smallest shard that would still let the remaining amount be + // delivered within the parts we have left. + minimum := intervalCeilDiv(maxAmt, partsLeft) + shards := p.shardAmounts(maxAmt, minimum, partsLeft) + + request := &intervalShardRequest{ + restrictions: restrictions, + shards: shards, + maxAmt: maxAmt, + minimum: minimum, + finalCltvDelta: finalCltvDelta, + finalHtlcExpiry: finalHtlcExpiry, + height: height, + } + + var ( + best *intervalChoice + pathErr noRouteError + found bool + ctx = context.TODO() + ) + + findBest := func(graph graphdb.NodeTraverser) error { + var err error + best, pathErr, found, err = p.chooseShard(ctx, graph, request) + + return err + } + + err := p.graphSessFactory.GraphSession(ctx, findBest, func() { + best, found = nil, false + }) + if err != nil { + return nil, err + } + + if best == nil { + // Report the most informative error the ladder produced, so + // that the payment is failed for the right reason. + if found { + return nil, pathErr + } + + return nil, errNoPathFound + } + + p.attempts++ + p.recordCapacities(best.edges, best.cache) + p.holdRoute(best.route) + + p.log.Debugf("Attempting shard of %v out of %v remaining over %v hops", + best.shard, maxAmt, len(best.route.Hops)) + + return best.route, nil +} + +// intervalShardRequest gathers what one call to RequestRoute needs to price +// every rung of its shard ladder. +type intervalShardRequest struct { + // restrictions are the constraints every route must respect. + restrictions *RestrictParams + + // shards is the ladder of candidate shard sizes. + shards []lnwire.MilliSatoshi + + // maxAmt is the whole remaining amount of the payment, and minimum the + // smallest shard that could still deliver it within the parts left. + maxAmt, minimum lnwire.MilliSatoshi + + // finalCltvDelta is the expiry delta of the final hop, including the + // block padding. + finalCltvDelta uint16 + + // finalHtlcExpiry is the absolute expiry height of the final hop, and + // height the current block height. + finalHtlcExpiry int32 + height uint32 +} + +// chooseShard prices every rung of the shard ladder and returns the best +// pairing of shard and route. The second and third return values carry the most +// informative non-critical error the ladder produced, for the case where no +// rung produced a route at all. +func (p *intervalPaymentSession) chooseShard(ctx context.Context, + graph graphdb.NodeTraverser, req *intervalShardRequest) ( + *intervalChoice, noRouteError, bool, error) { + + bandwidthHints, err := p.getBandwidthHints(graph) + if err != nil { + return nil, 0, false, err + } + + // If our own channels cannot cover the remaining amount in total, no + // arrangement of shards can complete the payment, and sending the parts + // we can afford would only leave them held at the receiver. + _, total, err := getOutgoingBalance( + p.selfNode, p.outgoingChanMap(), bandwidthHints, graph, + ) + if err != nil { + return nil, 0, false, err + } + if total < req.maxAmt { + p.log.Debugf("Local balance of %v cannot cover %v", total, + req.maxAmt) + + return nil, errInsufficientBalance, true, nil + } + + params := &intervalPathParams{ + graph: &graphParams{ + graph: graph, + additionalEdges: p.additionalEdges, + bandwidthHints: bandwidthHints, + }, + restrictions: req.restrictions, + cfg: &p.cfg, + probability: p.edgeProbability, + self: p.selfNode, + source: p.selfNode, + target: p.payment.Target, + finalHtlcExpiry: req.finalHtlcExpiry, + + // Every rung of the ladder walks the same graph, and the graph + // session held open around this loop means it cannot change + // underneath us, so the reads that do not depend on the amount + // are paid for once between all of them. + cache: newIntervalGraphCache(), + } + + // The appetite for a large shard depends on how the payment is going. + appetite := intervalAppetiteDefault + switch { + case p.settledParts > 0: + appetite = intervalAppetiteCommitted + + case p.failedAttempts >= intervalStrugglingAfter: + appetite = intervalAppetiteStruggling + + case p.failedAttempts > 0: + appetite = intervalAppetiteCautious + } + + var ( + best *intervalChoice + pathErr noRouteError + found bool + ) + + // The budget belongs to the payment rather than to any one shard, so + // every rung is priced against the same rate. + params.feeRate = p.feeRate(req.restrictions.FeeLimit) + + for _, shard := range req.shards { + params.amt = shard + + pathEdges, risk, err := findIntervalPath(ctx, params) + if err != nil { + var routeErr noRouteError + if !errors.As(err, &routeErr) { + return nil, 0, false, err + } + + // An error about the destination applies to every rung + // of the ladder, so there is no point walking the rest + // of it. + if routeErr != errNoPathFound { + return nil, routeErr, true, nil + } + + pathErr, found = routeErr, true + + continue + } + + rt, err := newRoute( + p.selfNode, pathEdges, req.height, finalHopParams{ + amt: shard, + totalAmt: p.payment.Amount, + cltvDelta: req.finalCltvDelta, + records: p.payment.DestCustomRecords, + paymentAddr: p.payment.PaymentAddr, + metadata: p.payment.Metadata, + }, p.payment.BlindedPathSet, + ) + if err != nil { + return nil, 0, false, err + } + + // Skip a route this payment has already watched fail at this + // amount or a smaller one. + if p.routeRejected(rt, shard) { + continue + } + + // Never hand out a route the payment cannot afford. The search + // prunes on the same limit while it walks, so reaching this is a + // sign that the route built from the path costs more than the + // path did, and the safe answer is to drop the rung rather than + // to spend an HTLC finding out. + if fee := rt.TotalAmount - shard; fee > req.restrictions.FeeLimit { + p.log.Debugf("Discarding a %v shard whose fee of %v "+ + "exceeds the remaining budget of %v", shard, + fee, req.restrictions.FeeLimit) + + continue + } + + choice := &intervalChoice{ + route: rt, + edges: pathEdges, + cache: params.cache, + shard: shard, + utility: intervalUtility( + rt, shard, req.maxAmt, req.minimum, + params.feeRate, risk, appetite, + ), + } + + // Ties break towards the larger shard, which keeps the router + // from cutting a payment finer than it has a reason to. + switch { + case best == nil: + best = choice + + case choice.utility > best.utility+1e-12: + best = choice + + case math.Abs(choice.utility-best.utility) <= 1e-12 && + choice.shard > best.shard: + + best = choice + } + } + + return best, pathErr, found, nil +} + +// intervalUtility prices a shard and the route found for it. The dominant term +// is the risk of the route, traded against how much of the remaining amount the +// shard would carry. +func intervalUtility(rt *route.Route, shard, maxAmt, + minimum lnwire.MilliSatoshi, feeRate intervalFeeRate, + risk, appetite float64) float64 { + + progress := math.Log(math.Max( + float64(shard)/math.Max(float64(minimum), 1), 1, + )) + + fee := rt.TotalAmount - shard + feePenalty := feeRate.penalty( + float64(fee), shard, intervalShardFeeWeight, + ) + hopPenalty := intervalShardHopWeight * float64(len(rt.Hops)) + + completionBonus := float64(0) + if shard == maxAmt { + completionBonus = intervalCompletionBonus + } + + return -risk + appetite*progress + completionBonus - feePenalty - + hopPenalty +} + +// partsLeft returns how many more HTLCs this payment is allowed to have in +// flight, given how many it has now. +func (p *intervalPaymentSession) partsLeft(activeShards uint32) uint32 { + maxParts := p.payment.MaxParts + if maxParts == 0 { + maxParts = 1 + } + + // Splitting also needs the receiver to be able to reassemble the parts. + if !p.canSplit() { + maxParts = 1 + } + + if activeShards >= maxParts { + return 0 + } + + return maxParts - activeShards +} + +// canSplit reports whether this payment may be cut into more than one HTLC. The +// conditions are the ones the stock session applies before it halves an amount: +// the receiver needs to be told what the parts add up to, which needs either a +// payment address or a blinded path, and it needs to understand MPP or AMP. +func (p *intervalPaymentSession) canSplit() bool { + if p.payment.PaymentAddr.IsNone() && p.payment.BlindedPathSet == nil { + return false + } + + features := p.payment.DestFeatures + if features == nil { + return false + } + + return features.HasFeature(lnwire.MPPOptional) || + features.HasFeature(lnwire.AMPOptional) +} + +// outgoingChanMap returns the first hop restriction as a set, or nil when the +// payment does not restrict its first hop. +func (p *intervalPaymentSession) outgoingChanMap() map[uint64]struct{} { + if len(p.payment.OutgoingChannelIDs) == 0 { + return nil + } + + chans := make(map[uint64]struct{}, len(p.payment.OutgoingChannelIDs)) + for _, chanID := range p.payment.OutgoingChannelIDs { + chans[chanID] = struct{}{} + } + + return chans +} + +// intervalCeilDiv divides an amount by a divisor, rounding up, so that the +// resulting shards always add up to at least the amount. +func intervalCeilDiv(amt lnwire.MilliSatoshi, + divisor uint32) lnwire.MilliSatoshi { + + if divisor <= 1 { + return amt + } + + d := lnwire.MilliSatoshi(divisor) + result := amt / d + if amt%d != 0 { + result++ + } + + return result +} + +// shardAmounts enumerates the shard sizes worth pricing for the given remaining +// amount. Four sources feed it: the amounts this payment has already proven do +// not fit, divided down until they do; the even division of the amount into a +// number of parts; the halving chain the stock session would walk one step at a +// time; and small multiples of the smallest usable shard. +// +// The first source is what makes the ladder a function of the beliefs rather +// than of the amount alone: a failure at some amount immediately puts shard +// sizes that sit just under it into play. It is enumerated first because every +// rung costs a full search, so when the ladder is cut short these are the rungs +// worth keeping. +func (p *intervalPaymentSession) shardAmounts(amt, + minimum lnwire.MilliSatoshi, partsLeft uint32) []lnwire.MilliSatoshi { + + if partsLeft <= 1 { + return []lnwire.MilliSatoshi{amt} + } + + limit := partsLeft + if limit > p.cfg.MaxShards { + limit = p.cfg.MaxShards + } + + var ( + seen = make(map[lnwire.MilliSatoshi]struct{}) + amounts = make([]lnwire.MilliSatoshi, 0, p.cfg.MaxLadderRungs) + ) + + add := func(shard lnwire.MilliSatoshi) { + if len(amounts) >= p.cfg.MaxLadderRungs { + return + } + + if shard == 0 || shard > amt || shard < minimum { + return + } + + // The whole remaining amount is always worth trying, but + // anything smaller has to clear the minimum shard size, since + // cutting below it produces HTLCs too small to be worth the + // round trip. + if shard != amt && shard < p.cfg.MinShardAmt { + return + } + + if _, ok := seen[shard]; ok { + return + } + + seen[shard] = struct{}{} + amounts = append(amounts, shard) + } + + add(amt) + add(minimum) + + for _, failedAt := range p.failedAt { + if failedAt <= 1 { + continue + } + + for _, divisor := range intervalShardDivisors { + add((failedAt - 1) / divisor) + } + } + + for parts := uint32(2); parts <= limit; parts++ { + add(intervalCeilDiv(amt, parts)) + } + + for shard := amt / 2; shard >= minimum && shard > 0; shard /= 2 { + add(shard) + + if shard == minimum { + break + } + } + + if minimum < amt { + for _, multiple := range intervalShardMultiples { + add(minimum * multiple) + } + } + + return amounts +} + +// edgeProbability prices a single hop for the path finder. It layers this +// payment's own experience over the node wide belief: a channel this payment +// has watched fail is discounted by how much smaller the retry is, and one it +// has a reason to distrust is discounted by the penalty it has accumulated. +func (p *intervalPaymentSession) edgeProbability(key IntervalKey, + amt, capacity lnwire.MilliSatoshi) float64 { + + if _, blocked := p.blocked[key]; blocked { + return 0 + } + + var probability float64 + + // Our own in-flight HTLCs have already committed part of what this edge + // had when we last looked at it, so a new shard of amt needs the edge + // to have held amt on top of what we are holding. Asking the model + // about the sum is the whole of the adjustment: it needs no new term, + // because every bound and every branch already answers the question + // "was there this much here". + // + // The first hop is the exception. The switch nets our in-flight HTLCs + // out of the bandwidth it reports for our own links, so the pathfinder + // has already been told, and adding the hold here would charge the same + // liquidity twice. + effective := amt + if key.From != p.selfNode { + effective += p.store.Held(key) + } + + failedAt := p.failedAt[key] + retryFactor := intervalRetryFactor(effective, failedAt) + if retryFactor == 0 { + return 0 + } + + if key.From == p.selfNode { + // We know our own balances exactly, and the bandwidth hints + // have already refused any channel that cannot carry the + // amount. The small haircut below certainty is what makes the + // search prefer a shorter route without a separate term for it. + probability = intervalLocalProbability + } else { + interval := p.store.Get(key, capacity) + probability = interval.Probability(effective, capacity) + + // A retry below an amount we have proven passes is not a retry + // at all, so the ladder does not apply to it. + if interval.LowerOK >= effective { + failedAt = 0 + } + } + + if probability == 0 { + return 0 + } + + if failedAt != 0 { + probability *= retryFactor + } + + if penalty := p.penalties[key]; penalty > 0 { + probability *= math.Exp( + -intervalPenaltyWeight * + math.Min(penalty, intervalPenaltyCap), + ) + } + + return math.Min( + math.Max(probability, intervalMinProbability), + intervalMaxProbability, + ) +} + +// ReportAttemptSuccess folds a settled shard into both the session's own state +// and the node wide belief. A settlement is the only observation that moves +// liquidity rather than merely bounding it. +// +// NOTE: Part of the PaymentResultReporter interface. +func (p *intervalPaymentSession) ReportAttemptSuccess(_ uint64, + rt *route.Route) { + + if rt == nil || len(rt.Hops) == 0 { + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + // The HTLC has resolved, so whatever it was holding is no longer held. + // The settlement recorded below moves the interval itself, which is how + // the liquidity this shard actually spent leaves our picture of the + // channel for good. + p.releaseRoute(rt) + + p.settledParts++ + + for i, key := range p.routeKeys(rt) { + amt := intervalHopAmount(rt, i) + + if key.From != p.selfNode { + p.store.RecordSettlement(key, amt, p.capacities[key]) + } + + // The session's own bounds describe the same channel, so they + // move with it. + p.shiftSessionLiquidity(key, amt) + + // A channel that just carried a part has earned back most of + // the suspicion this payment placed on it. + p.decayPenalty(key, intervalSettleDecay) + if p.suspects[key] > 1 { + p.suspects[key] /= 2 + } else { + delete(p.suspects, key) + } + } + + delete(p.routeFailedAt, intervalRouteKey(rt)) +} + +// ReportAttemptFailure folds a failed attempt into both the session's own state +// and the node wide belief. +// +// NOTE: Part of the PaymentResultReporter interface. +func (p *intervalPaymentSession) ReportAttemptFailure(_ uint64, rt *route.Route, + failureSourceIdx *int, failure lnwire.FailureMessage) { + + if rt == nil || len(rt.Hops) == 0 { + return + } + + p.mu.Lock() + defer p.mu.Unlock() + + // The HTLC has resolved, so whatever it was holding is no longer held. + p.releaseRoute(rt) + + p.failedAttempts++ + + keys := p.routeKeys(rt) + + // A failure we cannot attribute to any node, or one whose message we + // could not read, tells us only that something on this route went + // wrong. That is still worth something, and it is handled by + // elimination below rather than by penalizing the whole route. + if failureSourceIdx == nil || failure == nil { + p.recordUnattributedFailure(rt, keys) + + return + } + + failIndex := *failureSourceIdx + + // Every hop before the one that failed did forward, which proves it can + // carry the amount it was handed. + for i := 0; i < failIndex && i < len(keys); i++ { + key := keys[i] + if key.From == p.selfNode { + continue + } + + p.store.RecordProbe( + key, intervalHopAmount(rt, i), p.capacities[key], + ) + + p.decayPenalty(key, intervalProbeDecay) + if p.suspects[key] > 0 { + p.suspects[key]-- + } + } + + // A failure reported by the final node says nothing about the liquidity + // of any channel, so there is nothing to bound. An index outside the + // route should not be reachable, but a route we cannot index into is + // exactly the case where guessing would be worst. + if failIndex < 0 || failIndex >= len(keys) { + p.recordRouteFailure(rt, keys) + + return + } + + key := keys[failIndex] + amt := intervalHopAmount(rt, failIndex) + + switch failure.Code() { + // The channel is up but could not carry this amount, which is the one + // failure that carries a number we can bound with. + case lnwire.CodeTemporaryChannelFailure: + if key.From != p.selfNode { + p.store.RecordFailure(key, amt, p.capacities[key]) + } + + p.recordSessionFailure(key, amt) + + // The policy we hold for this channel is stale, so a retry would be + // built from the same stale policy. The channel update that came back + // with the failure has already been applied to the graph by the + // lifecycle, but this payment has no way to know whether it took, so it + // steps around the channel for the rest of its life. + case lnwire.CodeFeeInsufficient, lnwire.CodeIncorrectCltvExpiry: + p.blocked[key] = struct{}{} + p.penalties[key] += intervalPolicyPenalty + + default: + p.blocked[key] = struct{}{} + p.penalties[key] += intervalUnknownPenalty + } +} + +// recordUnattributedFailure does the attribution work that mission control +// hands to failPairRange, which penalizes every pair on the route because any +// of them could be to blame. Here the suspects are narrowed first: a hop we +// have already proven carries this amount cannot be the one that refused it. +// +// With one suspect left, elimination gives us a certainty for free. With none, +// something we believe is wrong and we say so with a flat penalty. With +// several, the suspicion is shared out and counted, and a channel that keeps +// turning up eventually gets treated as the cause. +func (p *intervalPaymentSession) recordUnattributedFailure(rt *route.Route, + keys []IntervalKey) { + + p.rejectRoute(rt) + + type suspect struct { + key IntervalKey + amt lnwire.MilliSatoshi + } + + suspects := make([]suspect, 0, len(keys)) + for i, key := range keys { + if key.From == p.selfNode { + continue + } + + // A hop we have already proven carries this amount cannot be + // the one that refused it. + amt := intervalHopAmount(rt, i) + if p.store.Get(key, p.capacities[key]).LowerOK >= amt { + continue + } + + suspects = append(suspects, suspect{key: key, amt: amt}) + } + + switch { + case len(suspects) == 1: + only := suspects[0] + p.store.RecordFailure( + only.key, only.amt, p.capacities[only.key], + ) + p.recordSessionFailure(only.key, only.amt) + + return + + case len(suspects) == 0: + for _, key := range keys { + p.penalties[key] += intervalContradictionPenalty + } + + return + } + + share := intervalSuspicionMass / math.Sqrt(float64(len(suspects))) + for _, item := range suspects { + p.suspects[item.key]++ + p.penalties[item.key] += share + + if p.suspects[item.key] >= intervalSuspicionEscalation { + p.penalties[item.key] += intervalExtraSuspicionPenalty + } + if p.suspects[item.key] >= intervalSuspicionCertainty { + p.boundSessionFailure(item.key, item.amt) + } + } +} + +// recordRouteFailure handles a failure that carries no liquidity information at +// all, such as one reported by the payment's own destination. +func (p *intervalPaymentSession) recordRouteFailure(rt *route.Route, + keys []IntervalKey) { + + p.rejectRoute(rt) + + for _, key := range keys { + p.penalties[key] += intervalRoutePenalty + } +} + +// rejectRoute records that this exact route failed at the amount it carried, so +// that it is not offered again at that amount or above. +func (p *intervalPaymentSession) rejectRoute(rt *route.Route) { + deliver := rt.ReceiverAmt() + routeKey := intervalRouteKey(rt) + + if previous := p.routeFailedAt[routeKey]; previous == 0 || + deliver < previous { + + p.routeFailedAt[routeKey] = deliver + } +} + +// routeRejected reports whether this payment already knows the given route +// cannot carry the given amount. +func (p *intervalPaymentSession) routeRejected(rt *route.Route, + deliver lnwire.MilliSatoshi) bool { + + failedAt := p.routeFailedAt[intervalRouteKey(rt)] + + return failedAt != 0 && deliver >= failedAt +} + +// recordSessionFailure notes that this payment watched a channel refuse an +// amount, and penalizes it for the rest of the payment. +func (p *intervalPaymentSession) recordSessionFailure(key IntervalKey, + amt lnwire.MilliSatoshi) { + + p.boundSessionFailure(key, amt) + p.penalties[key] += intervalFailurePenalty +} + +// boundSessionFailure lowers this payment's own upper bound for a channel. +func (p *intervalPaymentSession) boundSessionFailure(key IntervalKey, + amt lnwire.MilliSatoshi) { + + if previous := p.failedAt[key]; previous == 0 || amt < previous { + p.failedAt[key] = amt + } +} + +// shiftSessionLiquidity moves this payment's own bounds for a channel after a +// part has settled over it, mirroring what the store does to the beliefs. +func (p *intervalPaymentSession) shiftSessionLiquidity(key IntervalKey, + amt lnwire.MilliSatoshi) { + + if failedAt := p.failedAt[key]; failedAt != 0 { + if failedAt > amt { + p.failedAt[key] = failedAt - amt + } else { + p.failedAt[key] = 1 + } + } + + reverse := key.Reverse() + failedAt, ok := p.failedAt[reverse] + if !ok || failedAt == 0 { + return + } + + // Liquidity that just moved this way is liquidity the other direction + // gained, so a bound it held may no longer apply at all. + capacity := p.capacities[key] + if capacity != 0 && failedAt > capacity-amt { + delete(p.failedAt, reverse) + + return + } + + p.failedAt[reverse] = failedAt + amt +} + +// decayPenalty softens the penalty on a channel that has just proven itself, +// dropping it entirely once it no longer says anything. +func (p *intervalPaymentSession) decayPenalty(key IntervalKey, factor float64) { + if penalty := p.penalties[key]; penalty > intervalPenaltyFloor { + p.penalties[key] = penalty * factor + + return + } + + delete(p.penalties, key) +} + +// recordCapacities remembers the capacity each hop of a route was priced +// against, so that an outcome reported later can be recorded at the same scale. +// Without it we would have to go back to the graph on the failure path to learn +// what a channel's capacity was. +// +// NOTE: a route that did not come from this session, a resumed payment among +// them, leaves no capacity behind. The observations it produces are then +// dropped by the store, which is the right outcome, since an interval means +// nothing without the scale it is measured against. +func (p *intervalPaymentSession) recordCapacities(edges []*unifiedEdge, + cache *intervalGraphCache) { + + from := p.selfNode + for _, edge := range edges { + to := edge.policy.ToNodePubKey() + + key := IntervalKey{ + ChanID: edge.policy.ChannelID, + From: from, + To: to, + } + + // Record the hop under the same key it was priced under, so + // that what we learn from the attempt lands where the next + // search will look for it. + scoped := intervalScopeKey(key, cache.siblingCount(from, to)) + if scoped != key { + p.scopes[key] = scoped + } + + capacity := lnwire.NewMSatFromSatoshis(edge.capacity) + if capacity > 0 { + p.capacities[scoped] = capacity + } + + from = to + } +} + +// holdRoute records what a route we are about to hand out commits on each of +// its hops, and publishes it to the node wide overlay so that every other +// payment prices those hops knowing about it. +// +// NOTE: the first hop is skipped. The switch already nets our in-flight HTLCs +// out of the bandwidth it reports for our own links, so counting them here as +// well would charge the same liquidity twice. +func (p *intervalPaymentSession) holdRoute(rt *route.Route) { + amounts := make(map[IntervalKey]lnwire.MilliSatoshi) + for i, key := range p.routeKeys(rt) { + if key.From == p.selfNode { + continue + } + + amounts[key] += intervalHopAmount(rt, i) + } + + if len(amounts) == 0 { + return + } + + p.outstanding = append(p.outstanding, &heldShard{ + routeKey: intervalRouteKey(rt), + amounts: amounts, + }) + + p.store.Hold(amounts) +} + +// releaseRoute gives back what one resolved shard was holding. The oldest +// outstanding shard over the same route is the one released, since shards over +// an identical route are indistinguishable and resolve in the order they were +// sent often enough for this to be the better guess. +func (p *intervalPaymentSession) releaseRoute(rt *route.Route) { + routeKey := intervalRouteKey(rt) + + for i, shard := range p.outstanding { + if shard.routeKey != routeKey { + continue + } + + p.outstanding = append( + p.outstanding[:i], p.outstanding[i+1:]..., + ) + p.store.Release(shard.amounts) + + return + } +} + +// reconcileHolds drops the oldest holds until this session is holding no more +// shards than the payment has HTLCs in flight. +// +// The count comes from the payments database by way of the lifecycle, so it is +// ground truth, and this is what makes a hold impossible to leak while a +// payment is running. A route we returned that was never dispatched, because +// the traffic shaper or the database refused it after we handed it over, leaves +// a hold behind that no outcome will ever be reported for. Here it is dropped. +// +// Dropping the oldest is the safe direction. A hold that lingers depresses a +// channel for every payment on the node with nothing behind it, while a hold +// released early only costs us the contention we would have priced in. +func (p *intervalPaymentSession) reconcileHolds(activeShards uint32) { + for len(p.outstanding) > int(activeShards) { + stale := p.outstanding[0] + p.outstanding = p.outstanding[1:] + + p.store.Release(stale.amounts) + + p.log.Debugf("Released a hold on %d channels with no HTLC "+ + "behind it", len(stale.amounts)) + } +} + +// ReleaseAttempts gives back everything this session is still holding. The +// payment lifecycle calls it on the way out, which is the last moment anybody +// can, since a session is never reused once its lifecycle has returned. +// +// NOTE: Part of the PaymentResultReporter interface. +func (p *intervalPaymentSession) ReleaseAttempts() { + p.mu.Lock() + defer p.mu.Unlock() + + for _, shard := range p.outstanding { + p.store.Release(shard.amounts) + } + + p.outstanding = nil +} + +// feeRate returns how this session prices fees, given what the budget has left +// right now. +// +// The two halves of the answer come from different places on purpose. Whether +// there is a budget at all was latched when the session was built, from the +// limit the payment carries. How dearly a budget prices reliability comes from +// the remainder, which the lifecycle recomputes before every request and which +// therefore must never be asked whether a budget exists. +func (p *intervalPaymentSession) feeRate( + remaining lnwire.MilliSatoshi) intervalFeeRate { + + return newIntervalFeeRate(p.budgeted, remaining) +} + +// scoped returns the key a hop of a dispatched route was priced under. +func (p *intervalPaymentSession) scoped(key IntervalKey) IntervalKey { + if scoped, ok := p.scopes[key]; ok { + return scoped + } + + return key +} + +// routeKeys returns the key of every hop of a route, at the scope the hop was +// priced under. +func (p *intervalPaymentSession) routeKeys(rt *route.Route) []IntervalKey { + keys := intervalRouteKeys(rt) + for i, key := range keys { + keys[i] = p.scoped(key) + } + + return keys +} + +// intervalRouteKeys returns the directed channel key of every hop of a route. +func intervalRouteKeys(rt *route.Route) []IntervalKey { + keys := make([]IntervalKey, len(rt.Hops)) + + from := rt.SourcePubKey + for i, hop := range rt.Hops { + keys[i] = IntervalKey{ + ChanID: hop.ChannelID, + From: from, + To: hop.PubKeyBytes, + } + from = hop.PubKeyBytes + } + + return keys +} + +// intervalHopAmount returns the amount that flows over the given hop of a +// route, which is what the hop before it forwards. +func intervalHopAmount(rt *route.Route, hop int) lnwire.MilliSatoshi { + if hop == 0 { + return rt.TotalAmount + } + + return rt.Hops[hop-1].AmtToForward +} + +// intervalRouteKey identifies a route by the channels it walks, so that a route +// that has failed can be recognized when path finding produces it again. +func intervalRouteKey(rt *route.Route) string { + var key strings.Builder + + fmt.Fprintf(&key, "%x", rt.SourcePubKey[:]) + for _, hop := range rt.Hops { + fmt.Fprintf(&key, "/%d:%x", hop.ChannelID, hop.PubKeyBytes[:]) + } + + return key.String() +} diff --git a/routing/interval_session_source.go b/routing/interval_session_source.go new file mode 100644 index 00000000000..afe86897833 --- /dev/null +++ b/routing/interval_session_source.go @@ -0,0 +1,94 @@ +package routing + +import ( + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/htlcswitch" + "github.com/lightningnetwork/lnd/tlv" +) + +// A compile time assertion to ensure IntervalSessionSource meets the +// PaymentSessionSource interface. +var _ PaymentSessionSource = (*IntervalSessionSource)(nil) + +// IntervalSessionSource hands out interval router sessions. It wraps the stock +// session source rather than replacing it, both because the empty session it +// produces is a pure bookkeeping device with no routing in it, and because the +// interval router does not cover every payment shape yet and needs somewhere to +// fall back to. +type IntervalSessionSource struct { + // SessionSource is the stock source, used for the payment shapes the + // interval router does not handle. + *SessionSource + + // Store is the node wide liquidity belief the sessions read and write. + // It outlives every payment, which is what makes what one payment + // learns available to the next. + Store *IntervalStore + + // Config holds the search bounds of the interval router. + Config IntervalConfig +} + +// NewIntervalSessionSource builds a source that produces interval router +// sessions, backed by the given stock source for the payments it does not +// handle. +func NewIntervalSessionSource(stock *SessionSource, store *IntervalStore, + cfg IntervalConfig) *IntervalSessionSource { + + cfg.fillDefaults() + + return &IntervalSessionSource{ + SessionSource: stock, + Store: store, + Config: cfg, + } +} + +// NewPaymentSession creates a session for the given payment. Payments the +// interval router does not handle are served by the stock session instead, so +// that turning the router on never makes a payment unroutable that would +// otherwise have gone through. +// +// NOTE: Part of the PaymentSessionSource interface. +func (m *IntervalSessionSource) NewPaymentSession(p *LightningPayment, + firstHopBlob fn.Option[tlv.Blob], + trafficShaper fn.Option[htlcswitch.AuxTrafficShaper]) (PaymentSession, + error) { + + if reason := unsupportedByInterval(p); reason != "" { + log.Debugf("Payment %x falling back to the default router: %v", + p.Identifier(), reason) + + return m.SessionSource.NewPaymentSession( + p, firstHopBlob, trafficShaper, + ) + } + + getBandwidthHints := func(graph Graph) (bandwidthHints, error) { + return newBandwidthManager( + graph, m.SourceNode.PubKeyBytes, m.GetLink, + firstHopBlob, trafficShaper, + ) + } + + return newIntervalPaymentSession( + p, m.SourceNode.PubKeyBytes, getBandwidthHints, + m.GraphSessionFactory, m.Store, m.Config, + ) +} + +// unsupportedByInterval returns the reason the interval router cannot serve a +// payment, or the empty string when it can. +func unsupportedByInterval(p *LightningPayment) string { + // Blinded paths are served by the stock session. The interval model + // keys its beliefs on a directed channel, and inside a blinded path + // there is no channel to key on: the hops are opaque and the amounts + // and expiries of the intermediate ones are deliberately zero. Routing + // to the introduction node with intervals and through the path without + // them is a coherent design, but it is not this one. + if p.BlindedPathSet != nil { + return "payment is to a blinded path" + } + + return "" +} diff --git a/routing/interval_session_test.go b/routing/interval_session_test.go new file mode 100644 index 00000000000..911d2e5ac56 --- /dev/null +++ b/routing/interval_session_test.go @@ -0,0 +1,661 @@ +package routing + +import ( + "bytes" + "math" + "testing" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil/v2" + sphinx "github.com/lightningnetwork/lightning-onion" + "github.com/lightningnetwork/lnd/fn/v2" + "github.com/lightningnetwork/lnd/graph/db/models" + "github.com/lightningnetwork/lnd/htlcswitch" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/lightningnetwork/lnd/tlv" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// Node ids used by the interval session tests, on top of the source and target +// ids the mock graph already defines. +const ( + firstRelayID = 3 + secondRelayID = 4 +) + +// intervalTestCtx drives an interval session against the mock graph, standing +// in for the payment lifecycle: it asks for a route, sends it over the mock +// network, and reports the outcome back to the session the way the lifecycle +// does. +type intervalTestCtx struct { + t *testing.T + + graph *mockGraph + store *IntervalStore + session *intervalPaymentSession + payment *LightningPayment + + nextAttemptID uint64 +} + +// newIntervalTestCtx builds a context around a graph the caller has already +// populated. +func newIntervalTestCtx(t *testing.T, graph *mockGraph, + amt lnwire.MilliSatoshi, maxParts uint32, + splittable bool) *intervalTestCtx { + + t.Helper() + + payment := &LightningPayment{ + FinalCLTVDelta: 40, + FeeLimit: lnwire.MaxMilliSatoshi, + Target: graph.nodes[createPubkey(targetNodeID)].pubkey, + Amount: amt, + CltvLimit: math.MaxUint32, + MaxParts: maxParts, + } + + // A payment can only be split when the receiver can be told what the + // parts add up to, which needs both a payment address and a receiver + // that understands MPP. + if splittable { + var paymentAddr [32]byte + payment.PaymentAddr = fn.Some(paymentAddr) + payment.DestFeatures = lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + lnwire.PaymentAddrOptional, + lnwire.MPPOptional, + ), lnwire.Features, + ) + } else { + payment.DestFeatures = lnwire.NewFeatureVector( + lnwire.NewRawFeatureVector( + lnwire.TLVOnionPayloadOptional, + ), lnwire.Features, + ) + } + + var paymentHash [32]byte + require.NoError(t, payment.SetPaymentHash(paymentHash)) + + getBandwidthHints := func(_ Graph) (bandwidthHints, error) { + hints := map[uint64]lnwire.MilliSatoshi{} + for _, ch := range graph.source.channels { + hints[ch.id] = ch.balance + } + + return &mockBandwidthHints{hints: hints}, nil + } + + store := NewIntervalStore(0) + cfg := DefaultIntervalConfig() + + // The mock graph's channels are small, so shrink the minimum shard size + // to let the ladder cut them. + cfg.MinShardAmt = lnwire.NewMSatFromSatoshis(1000) + + session, err := newIntervalPaymentSession( + payment, graph.source.pubkey, getBandwidthHints, graph, store, + cfg, + ) + require.NoError(t, err) + + return &intervalTestCtx{ + t: t, + graph: graph, + store: store, + session: session, + payment: payment, + } +} + +// newIntervalTestGraph builds a graph with a source, a target and the given +// number of relays, each relay carrying a channel from the source and a channel +// to the target. +func newIntervalTestGraph(t *testing.T, relays []byte, + capacity btcutil.Amount) *mockGraph { + + t.Helper() + + graph := newMockGraph(t) + + source := newMockNode(sourceNodeID) + target := newMockNode(targetNodeID) + graph.addNode(source) + graph.addNode(target) + graph.source = source + + var chanID uint64 + for _, relay := range relays { + graph.addNode(newMockNode(relay)) + + chanID++ + graph.addChannel(chanID, sourceNodeID, relay, capacity) + + chanID++ + graph.addChannel(chanID, relay, targetNodeID, capacity) + } + + return graph +} + +// setBalance overrides the balance a node holds on its channel to a peer, which +// is how these tests arrange for a forward to fail. +func (c *intervalTestCtx) setBalance(node, peer byte, + balance lnwire.MilliSatoshi) { + + c.t.Helper() + + channel, ok := c.graph.nodes[createPubkey(node)]. + channels[createPubkey(peer)] + require.True(c.t, ok, "channel between %v and %v not found", node, peer) + + channel.balance = balance +} + +// attempt asks the session for a route for the given remaining amount, sends it +// over the mock network, and reports the outcome back. It returns the route and +// whether it settled. +func (c *intervalTestCtx) attempt(remaining lnwire.MilliSatoshi, + inFlight uint32) (*route.Route, bool, error) { + + c.t.Helper() + + rt, err := c.session.RequestRoute( + remaining, lnwire.MaxMilliSatoshi, inFlight, 0, nil, + ) + if err != nil { + return nil, false, err + } + + attemptID := c.nextAttemptID + c.nextAttemptID++ + + result, err := c.graph.sendHtlc(rt) + require.NoError(c.t, err) + + if result.failure == nil { + c.session.ReportAttemptSuccess(attemptID, rt) + + return rt, true, nil + } + + c.session.ReportAttemptFailure( + attemptID, rt, getNodeIndex(rt, result.failureSource), + result.failure, + ) + + return rt, false, nil +} + +// TestIntervalSessionFindsRoute tests that the session returns a usable route +// over a graph where one exists. +func TestIntervalSessionFindsRoute(t *testing.T) { + t.Parallel() + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, 100_000) + ctx := newIntervalTestCtx( + t, graph, lnwire.NewMSatFromSatoshis(10_000), 1, false, + ) + + rt, settled, err := ctx.attempt( + lnwire.NewMSatFromSatoshis(10_000), 0, + ) + require.NoError(t, err) + require.True(t, settled) + + // The only path is source, relay, target. + require.Len(t, rt.Hops, 2) + require.Equal( + t, createPubkey(firstRelayID), rt.Hops[0].PubKeyBytes, + ) + require.Equal(t, createPubkey(targetNodeID), rt.Hops[1].PubKeyBytes) + require.Equal( + t, lnwire.NewMSatFromSatoshis(10_000), rt.ReceiverAmt(), + ) + + // A settled route leaves the belief store holding what it moved, in + // both directions. + forward := IntervalKey{ + ChanID: 2, + From: createPubkey(firstRelayID), + To: createPubkey(targetNodeID), + } + capacity := lnwire.NewMSatFromSatoshis(100_000) + + require.True(t, ctx.store.Get(forward, capacity).Known) + require.True(t, ctx.store.Get(forward.Reverse(), capacity).Known) + require.GreaterOrEqual( + t, ctx.store.Get(forward.Reverse(), capacity).LowerOK, + lnwire.NewMSatFromSatoshis(10_000), + ) +} + +// TestIntervalSessionBoundsFailedChannel tests what a failure buys. The amount +// that failed becomes impossible rather than merely expensive, so the payment +// stops rather than retrying a route it now knows cannot work, and the bound +// outlives the payment: it is a smaller amount that the next one is free to +// try, not a channel that has been blacklisted. +func TestIntervalSessionBoundsFailedChannel(t *testing.T) { + t.Parallel() + + const capacitySat = 100_000 + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, capacitySat) + + amt := lnwire.NewMSatFromSatoshis(40_000) + ctx := newIntervalTestCtx(t, graph, amt, 1, false) + + // Starve the relay's channel to the target, so that it can forward a + // small amount but not the one we are about to send. + ctx.setBalance( + firstRelayID, targetNodeID, + lnwire.NewMSatFromSatoshis(5_000), + ) + + rt, settled, err := ctx.attempt(amt, 0) + require.NoError(t, err) + require.False(t, settled) + require.Equal(t, createPubkey(firstRelayID), rt.Hops[0].PubKeyBytes) + + // The failure left a bound in the store rather than a penalty that will + // fade, and the bound says the amount is impossible. + failed := IntervalKey{ + ChanID: 2, + From: createPubkey(firstRelayID), + To: createPubkey(targetNodeID), + } + capacity := lnwire.NewMSatFromSatoshis(capacitySat) + + interval := ctx.store.Get(failed, capacity) + require.Equal(t, amt, interval.UpperFail) + require.Zero(t, ctx.store.Probability(failed, amt, capacity)) + + // The payment cannot be split, so with its only route ruled out at this + // amount there is nothing left for it to try. + _, err = ctx.session.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.ErrorIs(t, err, errNoPathFound) + + // A bound is not a blacklist. A later payment reading the same store + // finds the channel perfectly usable below the amount that failed. + small := lnwire.NewMSatFromSatoshis(1_000) + require.Greater(t, ctx.store.Probability(failed, small, capacity), 0.0) + + next := newIntervalTestCtx(t, graph, small, 1, false) + next.store = ctx.store + next.session.store = ctx.store + + rt, settled, err = next.attempt(small, 0) + require.NoError(t, err) + require.True(t, settled) + require.Equal(t, small, rt.ReceiverAmt()) +} + +// TestIntervalSessionSplits tests that the session cuts a payment that no +// single channel can carry, choosing the shard size itself rather than halving +// its way down to one. +func TestIntervalSessionSplits(t *testing.T) { + t.Parallel() + + const capacitySat = 100_000 + + graph := newIntervalTestGraph( + t, []byte{firstRelayID, secondRelayID}, capacitySat, + ) + + // Each of our own channels holds half its capacity, so no single path + // can carry an amount above that, but the two together can. + amt := lnwire.NewMSatFromSatoshis(70_000) + ctx := newIntervalTestCtx(t, graph, amt, 3, true) + + remaining := amt + inFlight := uint32(0) + + var shards []lnwire.MilliSatoshi + for remaining > 0 { + require.Less(t, len(shards), 5, "too many shards") + + rt, settled, err := ctx.attempt(remaining, inFlight) + require.NoError(t, err) + require.True(t, settled) + + shards = append(shards, rt.ReceiverAmt()) + remaining -= rt.ReceiverAmt() + inFlight++ + } + + // The payment had to be split, and every shard was smaller than the + // whole. + require.Greater(t, len(shards), 1) + + var total lnwire.MilliSatoshi + for _, shard := range shards { + require.Less(t, shard, amt) + total += shard + } + require.Equal(t, amt, total) +} + +// TestIntervalSessionRefusesToSplit tests that a payment the receiver could not +// reassemble is never cut, no matter how many parts it allows. +func TestIntervalSessionRefusesToSplit(t *testing.T) { + t.Parallel() + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, 100_000) + + // This amount is above what our own channel holds, so the only way to + // deliver it would be to split, which this payment cannot do. + amt := lnwire.NewMSatFromSatoshis(70_000) + ctx := newIntervalTestCtx(t, graph, amt, 10, false) + + _, err := ctx.session.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.ErrorIs(t, err, errInsufficientBalance) +} + +// TestIntervalSessionAttemptLimit tests that a session which keeps finding +// routes still gives up eventually. +func TestIntervalSessionAttemptLimit(t *testing.T) { + t.Parallel() + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, 100_000) + + amt := lnwire.NewMSatFromSatoshis(10_000) + ctx := newIntervalTestCtx(t, graph, amt, 1, false) + ctx.session.cfg.AttemptLimit = 3 + + for i := 0; i < 3; i++ { + _, err := ctx.session.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + } + + _, err := ctx.session.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.ErrorIs(t, err, errNoPathFound) +} + +// TestIntervalShardAmounts tests the ladder of candidate shard sizes: it must +// stay inside the bounds the payment sets, and it must react to the amounts the +// payment has already proven do not fit. +func TestIntervalShardAmounts(t *testing.T) { + t.Parallel() + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, 100_000) + amt := lnwire.NewMSatFromSatoshis(100_000) + ctx := newIntervalTestCtx(t, graph, amt, 4, true) + + session := ctx.session + + // With a single part left, the only candidate is the whole amount. + require.Equal( + t, []lnwire.MilliSatoshi{amt}, + session.shardAmounts(amt, amt, 1), + ) + + // With four parts left, every candidate has to be large enough that + // four of them could still deliver the amount, and none may exceed it. + minimum := intervalCeilDiv(amt, 4) + amounts := session.shardAmounts(amt, minimum, 4) + + require.Contains(t, amounts, amt) + require.Contains(t, amounts, minimum) + for _, shard := range amounts { + require.GreaterOrEqual(t, shard, minimum) + require.LessOrEqual(t, shard, amt) + } + + // The ladder does not cut below the minimum shard size, other than to + // carry the whole remaining amount in one go. + small := session.cfg.MinShardAmt / 2 + require.Equal( + t, []lnwire.MilliSatoshi{small}, + session.shardAmounts(small, small/4, 4), + ) + + // An amount this payment has proven does not fit puts shard sizes just + // under it into play, which the ladder would otherwise never consider. + failedAt := amt*3/4 + 1 + session.failedAt[IntervalKey{ChanID: 1}] = failedAt + + withEvidence := session.shardAmounts(amt, minimum, 4) + require.Contains(t, withEvidence, (failedAt-1)/2) + require.NotContains(t, amounts, (failedAt-1)/2) + + // Every rung costs a full search, so the ladder is capped. The rungs + // that survive the cap are the ones enumerated first, which are the + // whole amount, the smallest usable shard, and the sizes the payment's + // own failures put into play. + session.cfg.MaxLadderRungs = 3 + capped := session.shardAmounts(amt, minimum, 4) + + require.Len(t, capped, 3) + require.Equal( + t, []lnwire.MilliSatoshi{amt, minimum, (failedAt - 1) / 2}, + capped, + ) +} + +// TestIntervalSessionSourceFallback tests that the payment shapes the interval +// router does not handle are served by the stock session instead, so that +// turning the router on cannot make a payment unroutable. +func TestIntervalSessionSourceFallback(t *testing.T) { + t.Parallel() + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, 100_000) + + stock := &SessionSource{ + GraphSessionFactory: graph, + SourceNode: &models.Node{ + PubKeyBytes: graph.source.pubkey, + }, + } + source := NewIntervalSessionSource( + stock, NewIntervalStore(0), IntervalConfig{}, + ) + + var paymentAddr [32]byte + payment := &LightningPayment{ + FinalCLTVDelta: 40, + Target: createPubkey(targetNodeID), + PaymentAddr: fn.Some(paymentAddr), + Amount: 1000, + CltvLimit: math.MaxUint32, + MaxParts: 1, + } + require.NoError(t, payment.SetPaymentHash([32]byte{})) + + // An ordinary payment is served by the interval session. + session, err := source.NewPaymentSession(payment, fn.None[tlv.Blob](), + fn.None[htlcswitch.AuxTrafficShaper]()) + require.NoError(t, err) + require.IsType(t, &intervalPaymentSession{}, session) + + // A payment to a blinded path falls back to the stock one, because the + // interval model has no directed channel to key its beliefs on inside a + // blinded path. The fallback has to be graceful: a payment lnd can route + // today must not become unroutable because this router is switched on. + // + // Route hints and a blinded path are mutually exclusive, so drop the + // hints the way a real blinded payment would arrive. + payment.RouteHints = nil + payment.BlindedPathSet = newTestBlindedPathSet(t) + + session, err = source.NewPaymentSession(payment, fn.None[tlv.Blob](), + fn.None[htlcswitch.AuxTrafficShaper]()) + require.NoError(t, err) + require.IsType(t, &paymentSession{}, session) + + // The fallback is transparent: it is exactly the session the stock + // source would have handed out on its own. + stockSession, err := stock.NewPaymentSession( + payment, fn.None[tlv.Blob](), + fn.None[htlcswitch.AuxTrafficShaper](), + ) + require.NoError(t, err) + require.IsType(t, stockSession, session) + + // A session that came from the fallback is the stock one all the way + // through, so it never reports attempts to a belief store that has no + // way to key them. + _, reports := session.(PaymentResultReporter) + require.False(t, reports) + + // The empty session is the stock one either way, since it holds no + // routing at all. + require.IsType(t, &paymentSession{}, source.NewPaymentSessionEmpty()) +} + +// recordingSession is a payment session that only records what the payment +// lifecycle tells it, so that the seam itself can be tested apart from the +// interval router that needed it. +type recordingSession struct { + PaymentSession + + successes []uint64 + failures []uint64 + released int +} + +// ReportAttemptSuccess records a settled attempt. +// +// NOTE: Part of the PaymentResultReporter interface. +func (r *recordingSession) ReportAttemptSuccess(attemptID uint64, + _ *route.Route) { + + r.successes = append(r.successes, attemptID) +} + +// ReportAttemptFailure records a failed attempt. +// +// NOTE: Part of the PaymentResultReporter interface. +func (r *recordingSession) ReportAttemptFailure(attemptID uint64, + _ *route.Route, _ *int, _ lnwire.FailureMessage) { + + r.failures = append(r.failures, attemptID) +} + +// ReleaseAttempts records that the lifecycle told the session it was done. +// +// NOTE: Part of the PaymentResultReporter interface. +func (r *recordingSession) ReleaseAttempts() { + r.released++ +} + +// TestLifecycleReportsToSession tests that the payment lifecycle hands an +// attempt outcome to a session that asked for it, on both the settle and the +// failure path, and that it does so alongside mission control rather than +// instead of it. +func TestLifecycleReportsToSession(t *testing.T) { + t.Parallel() + + p, m := newTestPaymentLifecycle(t) + + session := &recordingSession{PaymentSession: m.paySession} + p.paySession = session + + preimage := lntypes.Preimage{1} + attempt := makeSettledAttempt(t, 10_000, preimage) + + m.clock.On("Now").Return(time.Now()) + + // A settled attempt is reported to both mission control and the + // session. + m.missionControl.On("ReportPaymentSuccess", + attempt.AttemptID, &attempt.Route, + ).Return(nil).Once() + m.control.On("SettleAttempt", + p.identifier, attempt.AttemptID, mock.Anything, + ).Return(attempt, nil).Once() + + _, err := p.handleAttemptResult( + t.Context(), attempt, &htlcswitch.PaymentResult{ + Preimage: preimage, + }, + ) + require.NoError(t, err) + require.Equal(t, []uint64{attempt.AttemptID}, session.successes) + require.Empty(t, session.failures) + + // So is a failed one. An unreadable failure reaches mission control + // with neither a source nor a message, and the session hears about it + // on the same terms. + m.missionControl.On("ReportPaymentFail", + attempt.AttemptID, &attempt.Route, mock.Anything, mock.Anything, + ).Return(nil, nil).Once() + m.shardTracker.On("CancelShard", attempt.AttemptID).Return(nil).Once() + m.control.On("FailAttempt", + p.identifier, attempt.AttemptID, mock.Anything, + ).Return(attempt, nil).Once() + + _, err = p.handleSwitchErr( + t.Context(), attempt, htlcswitch.ErrUnreadableFailureMessage, + ) + require.NoError(t, err) + require.Equal(t, []uint64{attempt.AttemptID}, session.failures) +} + +// newTestBlindedPathSet builds a blinded path set that passes validation, so +// that the fallback is exercised on a payment lnd would really accept rather +// than on an empty struct. +func newTestBlindedPathSet(t *testing.T) *BlindedPaymentPathSet { + t.Helper() + + _, introPoint := btcec.PrivKeyFromBytes([]byte{1}) + _, blindedPoint := btcec.PrivKeyFromBytes([]byte{5}) + + payment := &BlindedPayment{ + BlindedPath: &sphinx.BlindedPath{ + IntroductionPoint: introPoint, + BlindingPoint: blindedPoint, + BlindedHops: []*sphinx.BlindedHopInfo{ + { + BlindedNodePub: introPoint, + CipherText: bytes.Repeat( + []byte{1}, 100, + ), + }, + }, + }, + BaseFee: 1000, + ProportionalFeeRate: 500, + CltvExpiryDelta: 140, + HtlcMinimum: 100, + HtlcMaximum: 100_000_000, + Features: lnwire.EmptyFeatureVector(), + } + + set, err := NewBlindedPaymentPathSet([]*BlindedPayment{payment}) + require.NoError(t, err) + + return set +} + +// TestStockSessionReportsNothing tests that with the interval router switched +// off, the seam it needed in the payment lifecycle is inert. The stock session +// does not implement the reporting interface, so the lifecycle's type assertion +// never fires and every attempt outcome goes to mission control and nowhere +// else, exactly as it did before. +func TestStockSessionReportsNothing(t *testing.T) { + t.Parallel() + + var session PaymentSession = &paymentSession{} + + _, reports := session.(PaymentResultReporter) + require.False(t, reports, "the stock session must stay inert") + + // The interval session is the one that asked for the seam. + session = &intervalPaymentSession{} + _, reports = session.(PaymentResultReporter) + require.True(t, reports) +} From 5ac19811d86ec6143f5abc42b53026e85bd58482 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:18:57 -0700 Subject: [PATCH 5/9] lnd+routerrpc: gate the interval router behind a config flag In this commit, we make the interval router selectable and leave it off. The new routerrpc.router option takes either default or interval, and defaults to default, so a node that says nothing about routing keeps exactly the stack it had before. With the flag off nothing in the new code is reached at all: the server builds the same session source it always did, and the seam the lifecycle grew is a type assertion the stock session does not satisfy. The interval source wraps the stock one rather than replacing it, so the payments the interval router does not serve still get a session that can route them. Both options are documented in the sample config, along with the conditions under which the second of them does anything. --- lnrpc/routerrpc/config.go | 12 ++++--- lnrpc/routerrpc/config_test.go | 58 +++++++++++++++++++++++++++++++ lnrpc/routerrpc/routing_config.go | 18 ++++++++++ sample-lnd.conf | 27 ++++++++++++++ server.go | 26 +++++++++++++- 5 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 lnrpc/routerrpc/config_test.go diff --git a/lnrpc/routerrpc/config.go b/lnrpc/routerrpc/config.go index 044c2436fc7..e9b7210ebbd 100644 --- a/lnrpc/routerrpc/config.go +++ b/lnrpc/routerrpc/config.go @@ -56,13 +56,15 @@ type Config struct { // DefaultConfig defines the config defaults. func DefaultConfig() *Config { defaultRoutingConfig := RoutingConfig{ + PaymentRouter: routing.DefaultPaymentRouter, ProbabilityEstimatorType: routing.DefaultEstimator, MinRouteProbability: routing.DefaultMinRouteProbability, - AttemptCost: routing.DefaultAttemptCost.ToSatoshis(), - AttemptCostPPM: routing.DefaultAttemptCostPPM, - MaxMcHistory: routing.DefaultMaxMcHistory, - McFlushInterval: routing.DefaultMcFlushInterval, + AttemptCost: routing.DefaultAttemptCost.ToSatoshis(), + AttemptCostPPM: routing.DefaultAttemptCostPPM, + MaxMcHistory: routing.DefaultMaxMcHistory, + McFlushInterval: routing.DefaultMcFlushInterval, + IntervalFlushInterval: routing.DefaultIntervalFlushInterval, AprioriConfig: &AprioriConfig{ HopProbability: routing.DefaultAprioriHopProbability, Weight: routing.DefaultAprioriWeight, @@ -85,12 +87,14 @@ func DefaultConfig() *Config { // GetRoutingConfig returns the routing config based on this sub server config. func GetRoutingConfig(cfg *Config) *RoutingConfig { return &RoutingConfig{ + PaymentRouter: cfg.PaymentRouter, ProbabilityEstimatorType: cfg.ProbabilityEstimatorType, MinRouteProbability: cfg.MinRouteProbability, AttemptCost: cfg.AttemptCost, AttemptCostPPM: cfg.AttemptCostPPM, MaxMcHistory: cfg.MaxMcHistory, McFlushInterval: cfg.McFlushInterval, + IntervalFlushInterval: cfg.IntervalFlushInterval, AprioriConfig: &AprioriConfig{ HopProbability: cfg.AprioriConfig.HopProbability, Weight: cfg.AprioriConfig.Weight, diff --git a/lnrpc/routerrpc/config_test.go b/lnrpc/routerrpc/config_test.go new file mode 100644 index 00000000000..1f2db84e621 --- /dev/null +++ b/lnrpc/routerrpc/config_test.go @@ -0,0 +1,58 @@ +package routerrpc + +import ( + "testing" + "time" + + "github.com/lightningnetwork/lnd/routing" + "github.com/stretchr/testify/require" +) + +// TestDefaultRouter tests that the payment router defaults to lnd's production +// stack, so that a node which says nothing about routing keeps the behaviour it +// had before the interval router existed. +func TestDefaultRouter(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + require.Equal(t, routing.DefaultPaymentRouter, cfg.PaymentRouter) + + // The selection survives the trip through GetRoutingConfig, which is + // what the server actually reads. + require.Equal( + t, routing.DefaultPaymentRouter, + GetRoutingConfig(cfg).PaymentRouter, + ) + + cfg.PaymentRouter = routing.IntervalPaymentRouter + require.Equal( + t, routing.IntervalPaymentRouter, + GetRoutingConfig(cfg).PaymentRouter, + ) +} + +// TestIntervalFlushInterval tests that the interval router's flush cadence has +// a default of its own and survives the trip through GetRoutingConfig, so that +// tuning it does not mean tuning mission control's cadence as well. +func TestIntervalFlushInterval(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + require.Equal( + t, routing.DefaultIntervalFlushInterval, + cfg.IntervalFlushInterval, + ) + require.Equal( + t, routing.DefaultIntervalFlushInterval, + GetRoutingConfig(cfg).IntervalFlushInterval, + ) + + // The two cadences move independently. + cfg.IntervalFlushInterval = 42 * time.Second + routingCfg := GetRoutingConfig(cfg) + + require.Equal(t, 42*time.Second, routingCfg.IntervalFlushInterval) + require.Equal( + t, routing.DefaultMcFlushInterval, routingCfg.McFlushInterval, + ) +} diff --git a/lnrpc/routerrpc/routing_config.go b/lnrpc/routerrpc/routing_config.go index c8393ebff1d..0a54abc644f 100644 --- a/lnrpc/routerrpc/routing_config.go +++ b/lnrpc/routerrpc/routing_config.go @@ -10,6 +10,18 @@ import ( // //nolint:ll type RoutingConfig struct { + // PaymentRouter selects the routing algorithm used to send payments. + // The default router is Dijkstra over a probability estimator, with + // mission control behind it and the shard amount halved whenever no + // route is found. The interval router replaces all of that with per + // directed channel liquidity intervals, and plans the shard amount and + // the route together. + // + // NOTE: this is named PaymentRouter rather than Router because the + // router sub server config already carries a Router field holding the + // channel router itself. + PaymentRouter string `long:"router" choice:"default" choice:"interval" description:"Routing algorithm used to send payments. The interval router is experimental."` + // ProbabilityEstimatorType sets the estimator to use. ProbabilityEstimatorType string `long:"estimator" choice:"apriori" choice:"bimodal" description:"Probability estimator used for pathfinding." ` @@ -36,6 +48,12 @@ type RoutingConfig struct { // control state to the DB. McFlushInterval time.Duration `long:"mcflushinterval" description:"the timer interval to use to flush mission control state to the DB"` + // IntervalFlushInterval defines the timer interval to use to flush the + // interval router's liquidity beliefs to the DB. It is only used when + // the interval router is selected and the node runs the native SQL + // backend. + IntervalFlushInterval time.Duration `long:"intervalflushinterval" description:"the timer interval to use to flush the interval router's liquidity beliefs to the DB"` + // AprioriConfig defines parameters for the apriori probability. AprioriConfig *AprioriConfig `group:"apriori" namespace:"apriori" description:"configuration for the apriori pathfinding probability estimator"` diff --git a/sample-lnd.conf b/sample-lnd.conf index f881c1174e9..55ed0cfe217 100644 --- a/sample-lnd.conf +++ b/sample-lnd.conf @@ -1312,6 +1312,28 @@ [routerrpc] +; Routing algorithm used to send payments. Two routers are available: default +; and interval. +; +; The default router finds a route with Dijkstra over a probability estimator, +; keeps what it learns in mission control as a penalty per node pair that fades +; with time, and halves the amount of a payment whenever no route can be found +; for it. +; +; The interval router keeps a liquidity interval per directed channel instead, +; bounded below by the largest amount it has watched pass and above by the +; smallest it has watched fail, with no time decay anywhere. It plans the size +; of an MPP shard together with the route that carries it, rather than halving +; its way down after a failure, and it retries a channel at a lower amount +; rather than stepping around it. +; +; Note that the interval router is experimental. Payments to blinded paths are +; served by the default router regardless of this setting. +; Default: +; routerrpc.router=default +; Example: +; routerrpc.router=interval + ; Probability estimator used for pathfinding. Two estimators are available: ; apriori and bimodal. ; Note that the bimodal estimator is experimental. @@ -1329,6 +1351,11 @@ ; The time interval with which the MC store state is flushed to the database. ; routerrpc.mcflushinterval=1s +; The time interval with which the interval router's liquidity beliefs are +; flushed to the database. Only used when routerrpc.router=interval and the node +; runs with db.use-native-sql=true; ignored otherwise. +; routerrpc.intervalflushinterval=1s + ; Path to the router macaroon. ; Default: ; routerrpc.routermacaroonpath=~/.lnd/data/chain/bitcoin/${network}/router.macaroon diff --git a/server.go b/server.go index a0312bb0146..0144788f04e 100644 --- a/server.go +++ b/server.go @@ -1100,7 +1100,7 @@ func newServer(ctx context.Context, cfg *Config, listenAddrs []net.Addr, if err != nil { return nil, fmt.Errorf("error getting source node: %w", err) } - paymentSessionSource := &routing.SessionSource{ + stockSessionSource := &routing.SessionSource{ GraphSessionFactory: s.v1Graph, SourceNode: sourceNode, MissionControl: s.defaultMC, @@ -1108,6 +1108,30 @@ func newServer(ctx context.Context, cfg *Config, listenAddrs []net.Addr, PathFindingConfig: pathFindingConfig, } + // Select the routing algorithm used to send payments. The default is + // the production stack the session source above implements; the + // interval router is an alternative paradigm that replaces mission + // control with per directed channel liquidity intervals and plans MPP + // shard amounts together with the routes that carry them. + var paymentSessionSource routing.PaymentSessionSource = stockSessionSource + switch routingConfig.PaymentRouter { + case routing.DefaultPaymentRouter: + + case routing.IntervalPaymentRouter: + srvrLog.Infof("Using the experimental interval router for " + + "payments") + + paymentSessionSource = routing.NewIntervalSessionSource( + stockSessionSource, + routing.NewIntervalStore(routingConfig.MaxMcHistory), + routing.DefaultIntervalConfig(), + ) + + default: + return nil, fmt.Errorf("unknown router type %v", + routingConfig.PaymentRouter) + } + s.controlTower = routing.NewControlTower(dbs.PaymentsDB) strictPruning := cfg.Bitcoin.Node == "neutrino" || From 50800700b01191c264c90e91c0488d1a272ad81c Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:19:33 -0700 Subject: [PATCH 6/9] routing+sqldb: persist liquidity intervals to the native SQL store In this commit, we let the router keep what it has learned across a restart. Everything it knows about the network is gathered by attempting payments, so a cold start is not merely slower, it is paid for in failed HTLCs until the picture is rebuilt. A new table holds one direction of one channel per row, the store loads what was written down when the node starts, and it writes changes back as it goes. We flush on a ticker rather than writing through. One attempt writes both directions of every hop it touched, so writing through would put a handful of database round trips between an HTLC failing and the next route being chosen, which is the one path in the router a user waits on. Nothing here has to survive a crash to stay correct either: a belief that never reached disk is one the router rediscovers on its next attempt, at the cost of that attempt. The cadence has a knob of its own, so that tuning it does not mean tuning mission control's as well. What comes back is deliberately not what went in. A belief read from disk is marked restored and its confidence halved, and the probability model then refuses to let either of its bounds speak with certainty. That matters more here than the same problem would in mission control, because this model has no clock: a bound moves only when evidence arrives, and an amount the model calls impossible is never attempted, so the evidence that would correct a stale bound could never turn up. Two choices in the schema are worth calling out. Confidence is stored in parts per million rather than as a float, because nothing else in this schema has a floating point column and we would rather not be the first to find out how the two dialects differ on one. And a short channel id of eight zero bytes is reserved for a belief about a node pair as a whole, which is the shape the router needs when a pair has several channels and an observation cannot say which of them carried the payment. Persistence is only available on a node running the native SQL backend. Everywhere else the store runs exactly as it did, memory only, with no dirty tracking and no goroutine. --- routing/interval_store_sql.go | 291 ++++++++++++++++++ routing/interval_store_sql_test.go | 197 ++++++++++++ routing/interval_test_postgres.go | 22 ++ routing/interval_test_sqlite.go | 15 + server.go | 41 ++- sqldb/migrations.go | 5 + sqldb/sqlc/liquidity_intervals.sql.go | 141 +++++++++ .../000016_liquidity_intervals.down.sql | 2 + .../000016_liquidity_intervals.up.sql | 64 ++++ sqldb/sqlc/models.go | 14 + sqldb/sqlc/querier.go | 5 + sqldb/sqlc/queries/liquidity_intervals.sql | 41 +++ 12 files changed, 836 insertions(+), 2 deletions(-) create mode 100644 routing/interval_store_sql.go create mode 100644 routing/interval_store_sql_test.go create mode 100644 routing/interval_test_postgres.go create mode 100644 routing/interval_test_sqlite.go create mode 100644 sqldb/sqlc/liquidity_intervals.sql.go create mode 100644 sqldb/sqlc/migrations/000016_liquidity_intervals.down.sql create mode 100644 sqldb/sqlc/migrations/000016_liquidity_intervals.up.sql create mode 100644 sqldb/sqlc/queries/liquidity_intervals.sql diff --git a/routing/interval_store_sql.go b/routing/interval_store_sql.go new file mode 100644 index 00000000000..75d4d13119e --- /dev/null +++ b/routing/interval_store_sql.go @@ -0,0 +1,291 @@ +package routing + +import ( + "context" + "database/sql" + "encoding/binary" + "fmt" + "math" + "time" + + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/lightningnetwork/lnd/sqldb" + "github.com/lightningnetwork/lnd/sqldb/sqlc" +) + +// intervalConfidenceScale is the denominator used to store a confidence, which +// ranges from zero to one, as an integer number of parts per million. +const intervalConfidenceScale = 1_000_000 + +// SQLIntervalQueries is the subset of the generated query interface that the +// interval store needs. Keeping it narrow means the store depends only on the +// queries it actually issues. +type SQLIntervalQueries interface { + UpsertLiquidityInterval(ctx context.Context, + arg sqlc.UpsertLiquidityIntervalParams) error + + ListLiquidityIntervals(ctx context.Context, limit int32) ( + []sqlc.LiquidityInterval, error) + + PruneLiquidityIntervals(ctx context.Context, limit int32) error + + DeleteLiquidityIntervals(ctx context.Context) error +} + +// BatchedIntervalQueries is a version of SQLIntervalQueries capable of batched +// database operations. +type BatchedIntervalQueries interface { + SQLIntervalQueries + + sqldb.BatchedTx[SQLIntervalQueries] +} + +// SQLIntervalStore persists liquidity interval beliefs to a SQL database. It +// is the durable half of the interval router's memory; the in-memory +// IntervalStore is what the router actually reads while it routes. +type SQLIntervalStore struct { + db BatchedIntervalQueries + + clock clock.Clock +} + +// A compile time assertion to ensure the SQL store satisfies the persister +// contract, and that the generated queries satisfy the narrow interface above. +var _ IntervalPersister = (*SQLIntervalStore)(nil) +var _ SQLIntervalQueries = (*sqlc.Queries)(nil) + +// NewSQLIntervalStore builds a store backed by the given database. +func NewSQLIntervalStore(db *sqldb.BaseDB) *SQLIntervalStore { + executor := sqldb.NewTransactionExecutor( + db, func(tx *sql.Tx) SQLIntervalQueries { + return db.WithTx(tx) + }, + ) + + return &SQLIntervalStore{ + db: executor, + clock: clock.NewDefaultClock(), + } +} + +// FetchIntervals returns at most limit of the most recently written beliefs. +// +// NOTE: Part of the IntervalPersister interface. +func (s *SQLIntervalStore) FetchIntervals(ctx context.Context, limit int) ( + []PersistedInterval, error) { + + var intervals []PersistedInterval + + err := s.db.ExecTx(ctx, sqldb.ReadTxOpt(), + func(tx SQLIntervalQueries) error { + rows, err := tx.ListLiquidityIntervals( + ctx, boundedLimit(limit), + ) + if err != nil { + return err + } + + intervals = make([]PersistedInterval, 0, len(rows)) + for _, row := range rows { + entry, err := unmarshalInterval(row) + if err != nil { + // A row we cannot read is a belief we + // no longer hold, which costs an + // attempt to rediscover and nothing + // more. Say so and carry on rather than + // refusing to start. + log.Warnf("Skipping unreadable "+ + "liquidity interval: %v", err) + + continue + } + + intervals = append(intervals, entry) + } + + return nil + }, func() { + intervals = nil + }, + ) + if err != nil { + return nil, err + } + + return intervals, nil +} + +// StoreIntervals writes the given beliefs, replacing any already held for the +// same directed channels. +// +// NOTE: Part of the IntervalPersister interface. +func (s *SQLIntervalStore) StoreIntervals(ctx context.Context, + intervals []PersistedInterval) error { + + if len(intervals) == 0 { + return nil + } + + now := s.clock.Now().UTC() + + return s.db.ExecTx(ctx, sqldb.WriteTxOpt(), + func(tx SQLIntervalQueries) error { + for _, entry := range intervals { + err := tx.UpsertLiquidityInterval( + ctx, marshalInterval(entry, now), + ) + if err != nil { + return fmt.Errorf("unable to store "+ + "liquidity interval: %w", err) + } + } + + return nil + }, sqldb.NoOpReset, + ) +} + +// PruneIntervals drops all but the given number of most recently written +// beliefs. +// +// NOTE: Part of the IntervalPersister interface. +func (s *SQLIntervalStore) PruneIntervals(ctx context.Context, + keep int) error { + + return s.db.ExecTx(ctx, sqldb.WriteTxOpt(), + func(tx SQLIntervalQueries) error { + return tx.PruneLiquidityIntervals( + ctx, boundedLimit(keep), + ) + }, sqldb.NoOpReset, + ) +} + +// PurgeIntervals drops every stored belief. +// +// NOTE: Part of the IntervalPersister interface. +func (s *SQLIntervalStore) PurgeIntervals(ctx context.Context) error { + return s.db.ExecTx(ctx, sqldb.WriteTxOpt(), + func(tx SQLIntervalQueries) error { + return tx.DeleteLiquidityIntervals(ctx) + }, sqldb.NoOpReset, + ) +} + +// boundedLimit converts a row limit into the type the generated queries take, +// clamping it into range. +func boundedLimit(limit int) int32 { + if limit <= 0 { + return DefaultMaxIntervalHistory + } + if limit > math.MaxInt32 { + return math.MaxInt32 + } + + return int32(limit) +} + +// marshalInterval turns a belief into the row that represents it. +func marshalInterval(entry PersistedInterval, + now time.Time) sqlc.UpsertLiquidityIntervalParams { + + var scid [8]byte + binary.BigEndian.PutUint64(scid[:], entry.Key.ChanID) + + interval := entry.Interval + + confidence := int64( + math.Round(interval.Confidence * intervalConfidenceScale), + ) + confidence = min(max(confidence, 0), intervalConfidenceScale) + + return sqlc.UpsertLiquidityIntervalParams{ + Scid: scid[:], + FromNode: entry.Key.From[:], + ToNode: entry.Key.To[:], + LowerOkMsat: int64(interval.LowerOK), + UpperFailMsat: int64(interval.UpperFail), + EstimateMsat: int64(interval.Estimate), + ConfidencePpm: confidence, + Successes: int64(interval.Successes), + Failures: int64(interval.Failures), + LiquidityMode: int32(interval.Mode), + UpdatedAt: now, + } +} + +// unmarshalInterval turns a stored row back into a belief. +// +// NOTE: the Restored flag is deliberately not set here. It is the store that +// decides a belief is restored, when it seeds one in, so that this function +// stays a plain reading of what is on disk. +func unmarshalInterval(row sqlc.LiquidityInterval) (PersistedInterval, error) { + var entry PersistedInterval + + if len(row.Scid) != 8 { + return entry, fmt.Errorf("expected an 8 byte channel id, got "+ + "%d bytes", len(row.Scid)) + } + if len(row.FromNode) != route.VertexSize || + len(row.ToNode) != route.VertexSize { + + return entry, fmt.Errorf("expected %d byte node keys, got %d "+ + "and %d bytes", route.VertexSize, len(row.FromNode), + len(row.ToNode)) + } + + // A negative amount cannot have been written by this store, so a row + // carrying one has been tampered with or corrupted. + if row.LowerOkMsat < 0 || row.UpperFailMsat < 0 || + row.EstimateMsat < 0 { + + return entry, fmt.Errorf("liquidity interval holds a negative " + + "amount") + } + + key := IntervalKey{ + ChanID: binary.BigEndian.Uint64(row.Scid), + } + copy(key.From[:], row.FromNode) + copy(key.To[:], row.ToNode) + + mode := int8(intervalModeUnknown) + switch { + case row.LiquidityMode < 0: + mode = intervalModeDepleted + + case row.LiquidityMode > 0: + mode = intervalModeRich + } + + confidence := float64(row.ConfidencePpm) / intervalConfidenceScale + + return PersistedInterval{ + Key: key, + Interval: LiquidityInterval{ + LowerOK: lnwire.MilliSatoshi(row.LowerOkMsat), + UpperFail: lnwire.MilliSatoshi(row.UpperFailMsat), + Estimate: lnwire.MilliSatoshi(row.EstimateMsat), + Confidence: min(max(confidence, 0), 1), + Successes: boundedCount(row.Successes), + Failures: boundedCount(row.Failures), + Mode: mode, + Known: true, + }, + }, nil +} + +// boundedCount converts a stored observation count back into the counter type, +// clamping it into range. +func boundedCount(count int64) uint32 { + if count <= 0 { + return 0 + } + if count > math.MaxUint32 { + return math.MaxUint32 + } + + return uint32(count) +} diff --git a/routing/interval_store_sql_test.go b/routing/interval_store_sql_test.go new file mode 100644 index 00000000000..f5a261ec2b1 --- /dev/null +++ b/routing/interval_store_sql_test.go @@ -0,0 +1,197 @@ +//go:build test_db_postgres || test_db_sqlite + +package routing + +import ( + "testing" + "time" + + "github.com/lightningnetwork/lnd/clock" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +// TestSQLIntervalStoreRoundTrip tests that a belief survives a trip through the +// database unchanged in everything the model reads. +func TestSQLIntervalStoreRoundTrip(t *testing.T) { + t.Parallel() + + store := NewSQLIntervalStore(newIntervalTestDB(t)) + ctx := t.Context() + + written := []PersistedInterval{ + { + Key: testIntervalKey, + Interval: LiquidityInterval{ + LowerOK: 1000, + UpperFail: 5000, + Estimate: 2500, + Confidence: 0.94, + Successes: 3, + Failures: 1, + Mode: intervalModeDepleted, + Known: true, + }, + }, + { + Key: testIntervalKey.Reverse(), + Interval: LiquidityInterval{ + LowerOK: 7000, + Estimate: 9000, + Confidence: 0.5, + Successes: 1, + Mode: intervalModeRich, + Known: true, + }, + }, + } + + require.NoError(t, store.StoreIntervals(ctx, written)) + + read, err := store.FetchIntervals(ctx, 100) + require.NoError(t, err) + require.Len(t, read, 2) + + byKey := make(map[IntervalKey]LiquidityInterval) + for _, entry := range read { + byKey[entry.Key] = entry.Interval + } + + for _, entry := range written { + got, ok := byKey[entry.Key] + require.True(t, ok, "missing key %v", entry.Key) + + require.Equal(t, entry.Interval.LowerOK, got.LowerOK) + require.Equal(t, entry.Interval.UpperFail, got.UpperFail) + require.Equal(t, entry.Interval.Estimate, got.Estimate) + require.Equal(t, entry.Interval.Successes, got.Successes) + require.Equal(t, entry.Interval.Failures, got.Failures) + require.Equal(t, entry.Interval.Mode, got.Mode) + require.True(t, got.Known) + + // Confidence goes to disk as parts per million, so it comes + // back to within one part in a million of itself. + require.InDelta( + t, entry.Interval.Confidence, got.Confidence, 1e-6, + ) + + // Nothing read from disk claims to be freshly observed. It is + // the in-memory store that decides that when it seeds a belief + // in, and it is what stops a restored bound from returning a + // hard zero. + require.False(t, got.Restored) + } + + // Writing the same directed channel again replaces the belief rather + // than adding a second row for it. + replaced := written[0] + replaced.Interval.UpperFail = 4000 + require.NoError( + t, store.StoreIntervals(ctx, []PersistedInterval{replaced}), + ) + + read, err = store.FetchIntervals(ctx, 100) + require.NoError(t, err) + require.Len(t, read, 2) + + for _, entry := range read { + if entry.Key == replaced.Key { + require.EqualValues(t, 4000, entry.Interval.UpperFail) + } + } + + // Purging leaves nothing behind. + require.NoError(t, store.PurgeIntervals(ctx)) + + read, err = store.FetchIntervals(ctx, 100) + require.NoError(t, err) + require.Empty(t, read) +} + +// TestSQLIntervalStorePrune tests that the table can be held to a bound. +func TestSQLIntervalStorePrune(t *testing.T) { + t.Parallel() + + store := NewSQLIntervalStore(newIntervalTestDB(t)) + ctx := t.Context() + + // Write in batches under a clock we step ourselves, so that the rows + // carry distinct timestamps whatever resolution the backend stores + // them at. Pruning keeps the most recently written. + testClock := clock.NewTestClock(time.Unix(1_700_000_000, 0).UTC()) + store.clock = testClock + + const batches = 4 + for i := 0; i < batches; i++ { + key := IntervalKey{ + ChanID: uint64(i), + From: route.Vertex{byte(i)}, + To: route.Vertex{byte(i), 1}, + } + + require.NoError(t, store.StoreIntervals( + ctx, []PersistedInterval{{ + Key: key, + Interval: LiquidityInterval{ + Known: true, + UpperFail: lnwire.MilliSatoshi(i + 1), + }, + }}, + )) + + testClock.SetTime(testClock.Now().Add(time.Minute)) + } + + require.NoError(t, store.PruneIntervals(ctx, 2)) + + read, err := store.FetchIntervals(ctx, 100) + require.NoError(t, err) + require.LessOrEqual(t, len(read), 2) + require.NotEmpty(t, read) +} + +// TestSQLIntervalStoreRestoresIntoMemory tests the whole path the router +// actually uses: beliefs written down by one store are read back by the next +// one to start, and arrive as soft evidence rather than as certainties. +func TestSQLIntervalStoreRestoresIntoMemory(t *testing.T) { + t.Parallel() + + db := newIntervalTestDB(t) + ctx := t.Context() + capacity := testIntervalCapacity + amt := capacity / 2 + + // A node runs, learns that an amount does not fit, and shuts down. + before := NewIntervalStore(0) + before.UsePersistence(NewSQLIntervalStore(db), time.Millisecond) + require.NoError(t, before.Start(ctx)) + + before.RecordFailure(testIntervalKey, amt, capacity) + require.Zero(t, before.Probability(testIntervalKey, amt, capacity)) + + require.NoError(t, before.Stop()) + + // A new node starts against the same database. + after := NewIntervalStore(0) + after.UsePersistence(NewSQLIntervalStore(db), time.Millisecond) + require.NoError(t, after.Start(ctx)) + t.Cleanup(func() { + require.NoError(t, after.Stop()) + }) + + interval := after.Get(testIntervalKey, capacity) + require.True(t, interval.Known) + require.True(t, interval.Restored) + require.Equal(t, amt, interval.UpperFail) + + // The bound came back, but it is no longer allowed to say impossible, + // so the amount can be attempted again and the belief corrected. + probability := after.Probability(testIntervalKey, amt, capacity) + require.GreaterOrEqual(t, probability, intervalRestoredFloor) + require.Less(t, probability, 0.5) + + // The reverse direction was written by the same observation and comes + // back too. + require.True(t, after.Get(testIntervalKey.Reverse(), capacity).Known) +} diff --git a/routing/interval_test_postgres.go b/routing/interval_test_postgres.go new file mode 100644 index 00000000000..17b4a729157 --- /dev/null +++ b/routing/interval_test_postgres.go @@ -0,0 +1,22 @@ +//go:build test_db_postgres && !test_db_sqlite + +package routing + +import ( + "testing" + + "github.com/lightningnetwork/lnd/sqldb" +) + +// newIntervalTestDB creates a Postgres backed database for the interval store +// tests. +func newIntervalTestDB(t testing.TB) *sqldb.BaseDB { + pgFixture := sqldb.NewTestPgFixture( + t, sqldb.DefaultPostgresFixtureLifetime, + ) + t.Cleanup(func() { + pgFixture.TearDown(t) + }) + + return sqldb.NewTestPostgresDB(t, pgFixture).GetBaseDB() +} diff --git a/routing/interval_test_sqlite.go b/routing/interval_test_sqlite.go new file mode 100644 index 00000000000..76a4b340577 --- /dev/null +++ b/routing/interval_test_sqlite.go @@ -0,0 +1,15 @@ +//go:build !test_db_postgres && test_db_sqlite + +package routing + +import ( + "testing" + + "github.com/lightningnetwork/lnd/sqldb" +) + +// newIntervalTestDB creates a SQLite backed database for the interval store +// tests. +func newIntervalTestDB(t testing.TB) *sqldb.BaseDB { + return sqldb.NewTestSqliteDB(t).GetBaseDB() +} diff --git a/server.go b/server.go index 0144788f04e..18f0ab7a989 100644 --- a/server.go +++ b/server.go @@ -364,6 +364,10 @@ type server struct { missionController *routing.MissionController defaultMC *routing.MissionControl + // intervalStore holds the liquidity beliefs of the interval router. It + // is nil unless that router is the one selected. + intervalStore *routing.IntervalStore + graphBuilder *graph.Builder chanRouter *routing.ChannelRouter @@ -1121,9 +1125,27 @@ func newServer(ctx context.Context, cfg *Config, listenAddrs []net.Addr, srvrLog.Infof("Using the experimental interval router for " + "payments") + s.intervalStore = routing.NewIntervalStore( + routingConfig.MaxMcHistory, + ) + + // The beliefs are worth keeping across a restart, but only a + // node running the native SQL backend has anywhere to keep + // them. Everywhere else the router starts cold, which costs it + // the attempts it takes to relearn the network. + if cfg.DB.UseNativeSQL && dbs.NativeSQLStore != nil { + srvrLog.Infof("Persisting liquidity interval beliefs " + + "to the native SQL store") + + s.intervalStore.UsePersistence( + routing.NewSQLIntervalStore( + dbs.NativeSQLStore.GetBaseDB(), + ), routingConfig.IntervalFlushInterval, + ) + } + paymentSessionSource = routing.NewIntervalSessionSource( - stockSessionSource, - routing.NewIntervalStore(routingConfig.MaxMcHistory), + stockSessionSource, s.intervalStore, routing.DefaultIntervalConfig(), ) @@ -2552,6 +2574,14 @@ func (s *server) Start(ctx context.Context) error { }) s.missionController.RunStoreTickers() + if s.intervalStore != nil { + cleanup = cleanup.add(s.intervalStore.Stop) + if err := s.intervalStore.Start(ctx); err != nil { + startErr = err + return + } + } + // Before we start the connMgr, we'll check to see if we have // any backups to recover. We do this now as we want to ensure // that have all the information we need to handle channel @@ -2898,6 +2928,13 @@ func (s *server) Stop() error { } s.missionController.StopStoreTickers() + if s.intervalStore != nil { + if err := s.intervalStore.Stop(); err != nil { + srvrLog.Warnf("Unable to stop interval "+ + "store: %v", err) + } + } + // Disconnect from each active peers to ensure that // peerTerminationWatchers signal completion to each peer. for _, peer := range s.Peers() { diff --git a/sqldb/migrations.go b/sqldb/migrations.go index 241e5c0d683..4c57769a2ab 100644 --- a/sqldb/migrations.go +++ b/sqldb/migrations.go @@ -136,6 +136,11 @@ var ( Version: 18, SchemaVersion: 15, }, + { + Name: "000016_liquidity_intervals", + Version: 19, + SchemaVersion: 16, + }, }, migrationAdditions...) // ErrMigrationMismatch is returned when a migrated record does not diff --git a/sqldb/sqlc/liquidity_intervals.sql.go b/sqldb/sqlc/liquidity_intervals.sql.go new file mode 100644 index 00000000000..c4b21dd9a0b --- /dev/null +++ b/sqldb/sqlc/liquidity_intervals.sql.go @@ -0,0 +1,141 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: liquidity_intervals.sql + +package sqlc + +import ( + "context" + "time" +) + +const countLiquidityIntervals = `-- name: CountLiquidityIntervals :one +SELECT COUNT(*) +FROM liquidity_intervals +` + +func (q *Queries) CountLiquidityIntervals(ctx context.Context) (int64, error) { + row := q.db.QueryRowContext(ctx, countLiquidityIntervals) + var count int64 + err := row.Scan(&count) + return count, err +} + +const deleteLiquidityIntervals = `-- name: DeleteLiquidityIntervals :exec +DELETE FROM liquidity_intervals +` + +func (q *Queries) DeleteLiquidityIntervals(ctx context.Context) error { + _, err := q.db.ExecContext(ctx, deleteLiquidityIntervals) + return err +} + +const listLiquidityIntervals = `-- name: ListLiquidityIntervals :many +SELECT scid, from_node, to_node, lower_ok_msat, upper_fail_msat, estimate_msat, confidence_ppm, successes, failures, liquidity_mode, updated_at +FROM liquidity_intervals +ORDER BY updated_at DESC +LIMIT $1 +` + +func (q *Queries) ListLiquidityIntervals(ctx context.Context, limit int32) ([]LiquidityInterval, error) { + rows, err := q.db.QueryContext(ctx, listLiquidityIntervals, limit) + if err != nil { + return nil, err + } + defer rows.Close() + var items []LiquidityInterval + for rows.Next() { + var i LiquidityInterval + if err := rows.Scan( + &i.Scid, + &i.FromNode, + &i.ToNode, + &i.LowerOkMsat, + &i.UpperFailMsat, + &i.EstimateMsat, + &i.ConfidencePpm, + &i.Successes, + &i.Failures, + &i.LiquidityMode, + &i.UpdatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const pruneLiquidityIntervals = `-- name: PruneLiquidityIntervals :exec +DELETE FROM liquidity_intervals +WHERE updated_at < ( + SELECT MIN(updated_at) + FROM ( + SELECT updated_at + FROM liquidity_intervals + ORDER BY updated_at DESC + LIMIT $1 + ) AS retained +) +` + +func (q *Queries) PruneLiquidityIntervals(ctx context.Context, limit int32) error { + _, err := q.db.ExecContext(ctx, pruneLiquidityIntervals, limit) + return err +} + +const upsertLiquidityInterval = `-- name: UpsertLiquidityInterval :exec +INSERT INTO liquidity_intervals ( + scid, from_node, to_node, lower_ok_msat, upper_fail_msat, estimate_msat, + confidence_ppm, successes, failures, liquidity_mode, updated_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 +) +ON CONFLICT (scid, from_node, to_node) DO UPDATE SET + lower_ok_msat = EXCLUDED.lower_ok_msat, + upper_fail_msat = EXCLUDED.upper_fail_msat, + estimate_msat = EXCLUDED.estimate_msat, + confidence_ppm = EXCLUDED.confidence_ppm, + successes = EXCLUDED.successes, + failures = EXCLUDED.failures, + liquidity_mode = EXCLUDED.liquidity_mode, + updated_at = EXCLUDED.updated_at +` + +type UpsertLiquidityIntervalParams struct { + Scid []byte + FromNode []byte + ToNode []byte + LowerOkMsat int64 + UpperFailMsat int64 + EstimateMsat int64 + ConfidencePpm int64 + Successes int64 + Failures int64 + LiquidityMode int32 + UpdatedAt time.Time +} + +func (q *Queries) UpsertLiquidityInterval(ctx context.Context, arg UpsertLiquidityIntervalParams) error { + _, err := q.db.ExecContext(ctx, upsertLiquidityInterval, + arg.Scid, + arg.FromNode, + arg.ToNode, + arg.LowerOkMsat, + arg.UpperFailMsat, + arg.EstimateMsat, + arg.ConfidencePpm, + arg.Successes, + arg.Failures, + arg.LiquidityMode, + arg.UpdatedAt, + ) + return err +} diff --git a/sqldb/sqlc/migrations/000016_liquidity_intervals.down.sql b/sqldb/sqlc/migrations/000016_liquidity_intervals.down.sql new file mode 100644 index 00000000000..a08ea330c73 --- /dev/null +++ b/sqldb/sqlc/migrations/000016_liquidity_intervals.down.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS idx_liquidity_intervals_updated_at; +DROP TABLE IF EXISTS liquidity_intervals; diff --git a/sqldb/sqlc/migrations/000016_liquidity_intervals.up.sql b/sqldb/sqlc/migrations/000016_liquidity_intervals.up.sql new file mode 100644 index 00000000000..227015ff93c --- /dev/null +++ b/sqldb/sqlc/migrations/000016_liquidity_intervals.up.sql @@ -0,0 +1,64 @@ +-- ───────────────────────────────────────────── +-- Liquidity Intervals +-- ───────────────────────────────────────────── +-- Stores what the interval router believes about the liquidity available in +-- one direction of one channel. Unlike mission control, which records a +-- penalty per node pair that fades with time, this is an amount interval: +-- bounded below by the largest amount the router has watched pass and above by +-- the smallest it has watched fail. +-- +-- A row is written per direction, so an ordinary channel occupies two rows +-- whose scid agrees and whose node columns are swapped. A scid of eight zero +-- bytes is reserved for a belief held about a node pair as a whole rather than +-- about a specific channel, which is what the router falls back to when a pair +-- has several channels between it and an observation cannot say which one +-- carried the payment. +-- ───────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS liquidity_intervals ( + -- The short channel id of the channel, as eight big endian bytes. All + -- zeroes means the row describes the node pair rather than one channel. + scid BLOB NOT NULL, + + -- The public key of the node the liquidity flows away from. + from_node BLOB NOT NULL, + + -- The public key of the node the liquidity flows towards. + to_node BLOB NOT NULL, + + -- The largest amount in millisatoshis this direction has been proven to + -- carry. Zero means nothing has been proven. + lower_ok_msat BIGINT NOT NULL, + + -- The smallest amount in millisatoshis this direction has been proven not + -- to carry. Zero means no failure has been observed. + upper_fail_msat BIGINT NOT NULL, + + -- The best guess in millisatoshis at the balance available. + estimate_msat BIGINT NOT NULL, + + -- How much evidence stands behind the estimate, in parts per million of a + -- confidence that ranges from zero to one. Stored as an integer because + -- the schema has no floating point column anywhere else. + confidence_ppm BIGINT NOT NULL, + + -- How many observations of each kind have landed on this direction. + successes BIGINT NOT NULL, + failures BIGINT NOT NULL, + + -- Which side of the bimodal liquidity distribution this direction appears + -- to sit on: -1 depleted, 0 unclassified, 1 saturated. + liquidity_mode INTEGER NOT NULL, + + -- When this belief was last written. Used only to decide which rows to + -- drop when the table outgrows its bound; the model itself has no clock + -- and never expires a belief because time has passed. + updated_at TIMESTAMP NOT NULL, + + PRIMARY KEY (scid, from_node, to_node) +); + +-- Index supporting the read of the most recently written beliefs at startup +-- and the pruning of the oldest ones. +CREATE INDEX IF NOT EXISTS idx_liquidity_intervals_updated_at +ON liquidity_intervals(updated_at); diff --git a/sqldb/sqlc/models.go b/sqldb/sqlc/models.go index ef9aa9006f9..b615d12e855 100644 --- a/sqldb/sqlc/models.go +++ b/sqldb/sqlc/models.go @@ -209,6 +209,20 @@ type InvoiceSequence struct { CurrentValue int64 } +type LiquidityInterval struct { + Scid []byte + FromNode []byte + ToNode []byte + LowerOkMsat int64 + UpperFailMsat int64 + EstimateMsat int64 + ConfidencePpm int64 + Successes int64 + Failures int64 + LiquidityMode int32 + UpdatedAt time.Time +} + type MigrationTracker struct { Version int32 MigrationTime time.Time diff --git a/sqldb/sqlc/querier.go b/sqldb/sqlc/querier.go index 9b95a669917..43dd8c4d1ab 100644 --- a/sqldb/sqlc/querier.go +++ b/sqldb/sqlc/querier.go @@ -15,6 +15,7 @@ type Querier interface { AddV1ChannelProof(ctx context.Context, arg AddV1ChannelProofParams) (sql.Result, error) AddV2ChannelProof(ctx context.Context, arg AddV2ChannelProofParams) (sql.Result, error) ClearKVInvoiceHashIndex(ctx context.Context) error + CountLiquidityIntervals(ctx context.Context) (int64, error) CountPayments(ctx context.Context) (int64, error) CountZombieChannels(ctx context.Context, version int16) (int64, error) CreateChannel(ctx context.Context, arg CreateChannelParams) (int64, error) @@ -28,6 +29,7 @@ type Querier interface { // resolutions across all payments. DeleteFailedAttempts(ctx context.Context, paymentID int64) error DeleteInvoice(ctx context.Context, arg DeleteInvoiceParams) (sql.Result, error) + DeleteLiquidityIntervals(ctx context.Context) error DeleteNode(ctx context.Context, id int64) error DeleteNodeAddresses(ctx context.Context, nodeID int64) error DeleteNodeByPubKey(ctx context.Context, arg DeleteNodeByPubKeyParams) (sql.Result, error) @@ -229,6 +231,7 @@ type Querier interface { ListChannelsPaginatedV2(ctx context.Context, arg ListChannelsPaginatedV2Params) ([]ListChannelsPaginatedV2Row, error) ListChannelsWithPoliciesForCachePaginated(ctx context.Context, arg ListChannelsWithPoliciesForCachePaginatedParams) ([]ListChannelsWithPoliciesForCachePaginatedRow, error) ListChannelsWithPoliciesPaginated(ctx context.Context, arg ListChannelsWithPoliciesPaginatedParams) ([]ListChannelsWithPoliciesPaginatedRow, error) + ListLiquidityIntervals(ctx context.Context, limit int32) ([]LiquidityInterval, error) ListNodeIDsAndPubKeys(ctx context.Context, arg ListNodeIDsAndPubKeysParams) ([]ListNodeIDsAndPubKeysRow, error) ListNodesPaginated(ctx context.Context, arg ListNodesPaginatedParams) ([]GraphNode, error) NextInvoiceSettleIndex(ctx context.Context) (int64, error) @@ -239,6 +242,7 @@ type Querier interface { OnInvoiceCanceled(ctx context.Context, arg OnInvoiceCanceledParams) error OnInvoiceCreated(ctx context.Context, arg OnInvoiceCreatedParams) error OnInvoiceSettled(ctx context.Context, arg OnInvoiceSettledParams) error + PruneLiquidityIntervals(ctx context.Context, limit int32) error SetKVInvoicePaymentHash(ctx context.Context, arg SetKVInvoicePaymentHashParams) error SetMigration(ctx context.Context, arg SetMigrationParams) error SettleAttempt(ctx context.Context, arg SettleAttemptParams) error @@ -252,6 +256,7 @@ type Querier interface { UpsertChanPolicyExtraType(ctx context.Context, arg UpsertChanPolicyExtraTypeParams) error UpsertChannelExtraType(ctx context.Context, arg UpsertChannelExtraTypeParams) error UpsertEdgePolicy(ctx context.Context, arg UpsertEdgePolicyParams) (int64, error) + UpsertLiquidityInterval(ctx context.Context, arg UpsertLiquidityIntervalParams) error UpsertNode(ctx context.Context, arg UpsertNodeParams) (int64, error) UpsertNodeAddress(ctx context.Context, arg UpsertNodeAddressParams) error UpsertNodeExtraType(ctx context.Context, arg UpsertNodeExtraTypeParams) error diff --git a/sqldb/sqlc/queries/liquidity_intervals.sql b/sqldb/sqlc/queries/liquidity_intervals.sql new file mode 100644 index 00000000000..59164ada065 --- /dev/null +++ b/sqldb/sqlc/queries/liquidity_intervals.sql @@ -0,0 +1,41 @@ +-- name: UpsertLiquidityInterval :exec +INSERT INTO liquidity_intervals ( + scid, from_node, to_node, lower_ok_msat, upper_fail_msat, estimate_msat, + confidence_ppm, successes, failures, liquidity_mode, updated_at +) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11 +) +ON CONFLICT (scid, from_node, to_node) DO UPDATE SET + lower_ok_msat = EXCLUDED.lower_ok_msat, + upper_fail_msat = EXCLUDED.upper_fail_msat, + estimate_msat = EXCLUDED.estimate_msat, + confidence_ppm = EXCLUDED.confidence_ppm, + successes = EXCLUDED.successes, + failures = EXCLUDED.failures, + liquidity_mode = EXCLUDED.liquidity_mode, + updated_at = EXCLUDED.updated_at; + +-- name: ListLiquidityIntervals :many +SELECT * +FROM liquidity_intervals +ORDER BY updated_at DESC +LIMIT $1; + +-- name: CountLiquidityIntervals :one +SELECT COUNT(*) +FROM liquidity_intervals; + +-- name: PruneLiquidityIntervals :exec +DELETE FROM liquidity_intervals +WHERE updated_at < ( + SELECT MIN(updated_at) + FROM ( + SELECT updated_at + FROM liquidity_intervals + ORDER BY updated_at DESC + LIMIT $1 + ) AS retained +); + +-- name: DeleteLiquidityIntervals :exec +DELETE FROM liquidity_intervals; From 61638661ae841c77d9b8a0d4f56130c8def7caf4 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:20:10 -0700 Subject: [PATCH 7/9] routing: quarantine a failure that cannot say where it happened In this commit, we give the router somewhere to put evidence it cannot trust. An unreadable onion error, or one that no node on the route claims, leaves several hops that could each have been the one to refuse. Writing an upper bound on all of them would be a claim of certainty about channels that may be perfectly healthy, and a bound in this model is permanent until evidence moves it: an amount it calls impossible is never attempted, so a wrong bound can never be corrected. Ambiguous evidence goes into a quarantine instead, held per directed channel alongside the interval but deliberately outside it. It keeps the smallest amount such a failure has named and how much corroboration stands behind it, where a failure naming two suspects contributes half of what one naming a single suspect would, so the more ways a failure could be explained the more of them it takes to convict. While it sits there it prices as a discount on the amount it named and never as an impossibility. Enough agreement promotes it into an ordinary upper bound. What may release a suspicion is the subtle part, and it is narrower than it first appears. When a hop reports a failure, the router writes a lower bound on every hop before it, because forwarding is what carried the payment that far. That inference holds exactly when the report names the right hop. When blame arrives shifted downstream, which is one of the ways a real network lies, the guilty channel sits before the reported index and collects a lower bound saying it carried the amount it had just refused. A quarantine that accepted lower bounds as proof of innocence would let that channel out of every suspicion it belonged in and pile the blame onto its innocent neighbours. So the router keeps a separate record of what it has watched actually move, written by nothing but a settlement, and only that clears a suspicion. The lower bound keeps every other job it has, in path finding and in bound maintenance, and loses only a standing that was never really its to give. Three smaller decisions come with it. A settlement proves the direction it moved over and no other, so the reverse direction is left alone. Neither the quarantine nor the record of settlements is written to disk, since a suspicion restored from a previous run is one nothing since could have cleared, and a settlement from before a restart says nothing about a failure observed after it. And there is exactly one place anything enters the quarantine, with a config field on it, so that a node can switch the whole mechanism off and get the payment-local handling it had before. --- routing/interval_belief.go | 183 +++++++++ routing/interval_config.go | 12 + routing/interval_quarantine_test.go | 596 ++++++++++++++++++++++++++++ routing/interval_session.go | 32 +- routing/interval_store.go | 37 ++ 5 files changed, 857 insertions(+), 3 deletions(-) create mode 100644 routing/interval_quarantine_test.go diff --git a/routing/interval_belief.go b/routing/interval_belief.go index ca252acb13f..2e838946257 100644 --- a/routing/interval_belief.go +++ b/routing/interval_belief.go @@ -122,6 +122,31 @@ const ( // a belief when it is restored, since whatever evidence stood behind it // is now at least one restart old. intervalRestoredConfidence = 0.5 + + // intervalSuspectPromoteWeight is the corroboration a quarantined + // observation needs before it becomes a bound. A failure that names two + // suspects contributes about 0.7 to each of them, so roughly three such + // failures agreeing on the same channel promote; one naming five + // suspects contributes 0.45, so five are needed. The more ambiguous a + // failure is, the more of them it takes to convict. + intervalSuspectPromoteWeight = 2.05 + + // intervalSuspectPenalty is how hard a quarantined observation prices, + // per unit of the weight standing behind it. A single ambiguous failure + // discounts an amount noticeably without ruling it out, which is the + // whole point of holding it apart from the bounds. + intervalSuspectPenalty = 0.55 + + // intervalSuspectPenaltyCap bounds the discount, so that a channel + // which keeps turning up in ambiguous failures without ever being + // convicted cannot be priced out of the graph entirely. + intervalSuspectPenaltyCap = 3.0 + + // intervalSuspectEstimateNumerator and + // intervalSuspectEstimateDenominator give the estimate a promoted + // suspicion leaves behind, as a fraction of the amount it named. + intervalSuspectEstimateNumerator = 68 + intervalSuspectEstimateDenominator = 100 ) // Liquidity mode classifications. The model does not just carry a probability @@ -327,6 +352,48 @@ type LiquidityInterval struct { // observation, because from that point the bounds describe evidence we // gathered ourselves. Restored bool + + // ProvenOK is the largest amount this direction has been watched + // actually move, which is to say the largest amount a payment settled + // over it. It is the only evidence class strong enough to clear a + // suspicion, and nothing but a settlement ever writes it. + // + // LowerOK is not that, which is the distinction this field exists to + // draw. LowerOK also rises when a failure reported by some hop implies + // that the hops before it forwarded, and under misattribution that + // implication is exactly what breaks: blame shifted downstream puts the + // guilty channel before the reported index, so it collects a lower bound + // claiming it carried the amount it had in fact just refused. Reading + // that as proof of innocence lets the culprit walk out of every + // suspicion it should have been held for. + // + // A settlement proves the forward direction and only the forward + // direction. It does move balance to the other side, which is why the + // reverse interval slides up, but sliding an interval is an inference + // about a balance and this field is a record of something watched. So + // the reverse direction is left alone. + // + // NOTE: this is not persisted. It says a settlement was watched by this + // process, and a settlement from before a restart is evidence about a + // network that has had the restart to move on. A failure observed now + // outranks it, so a restored belief starts with nothing here and earns + // it back with the first settlement. + ProvenOK lnwire.MilliSatoshi + + // SuspectAmt is the smallest amount that a failure we could not + // attribute has named for this channel, and SuspectWeight is how much + // corroboration those failures carry between them. Zero means nothing + // is under suspicion. + // + // This pair is the quarantine. An observation whose attribution we do + // not trust is held here rather than written into the bounds above, + // because a bound is a claim of certainty and an ambiguous failure is + // not one. Quarantined evidence prices as a discount and never as an + // impossibility. It is promoted into a real upper bound once enough + // independent failures agree on it, and it is cleared the moment the + // channel proves it can carry the amount after all. + SuspectAmt lnwire.MilliSatoshi + SuspectWeight float64 } // markRestored turns a belief loaded from disk into soft evidence. The bounds @@ -336,6 +403,12 @@ type LiquidityInterval struct { func (l *LiquidityInterval) markRestored() { l.Restored = true l.Confidence *= intervalRestoredConfidence + + // Proof of a settlement does not survive a restart. It is never written + // down, and a belief being seeded in must not carry one regardless of + // what the caller handed us, because a settlement from before the + // restart says nothing about a failure observed after it. + l.ProvenOK = 0 } // normalize restores the invariant 0 <= LowerOK <= Estimate < UpperFail <= @@ -368,6 +441,30 @@ func (l *LiquidityInterval) normalize(capacity lnwire.MilliSatoshi) { } } + if l.ProvenOK > capacity { + l.ProvenOK = capacity + } + + if l.SuspectAmt > capacity { + l.SuspectAmt = capacity + } + + // A channel we have watched settle the suspected amount is a channel the + // suspicion was wrong about. This is the contradiction rule, and putting + // it here means it fires no matter which settlement moved the bound. + // + // It reads ProvenOK rather than LowerOK on purpose. See ProvenOK. + if l.SuspectAmt != 0 && l.ProvenOK >= l.SuspectAmt { + l.clearSuspect() + } + + // A bound we do trust says everything the suspicion was reaching for. + if l.SuspectAmt != 0 && l.UpperFail != 0 && + l.UpperFail <= l.SuspectAmt { + + l.clearSuspect() + } + if capacity == 0 { return } @@ -383,6 +480,80 @@ func (l *LiquidityInterval) normalize(capacity lnwire.MilliSatoshi) { } } +// clearSuspect empties the quarantine. +func (l *LiquidityInterval) clearSuspect() { + l.SuspectAmt = 0 + l.SuspectWeight = 0 +} + +// recordSuspect quarantines a failure we cannot attribute with confidence. The +// weight says how much this one failure implicates this channel, which is a +// question of how many other channels it implicated equally. +// +// Nothing is written to the bounds until the weight standing behind the +// quarantine crosses the promotion threshold. At that point enough independent +// failures have agreed on the same channel and the same amount that treating it +// as proven is the better bet than continuing to guess. +func (l *LiquidityInterval) recordSuspect(amt, capacity lnwire.MilliSatoshi, + weight float64) { + + // An amount we have watched settle over this channel is not a suspicion + // worth holding. Only a settlement counts here; see ProvenOK. + if l.ProvenOK != 0 && l.ProvenOK >= amt { + return + } + + if l.SuspectAmt == 0 || amt < l.SuspectAmt { + l.SuspectAmt = amt + } + l.SuspectWeight += weight + + if l.SuspectWeight < intervalSuspectPromoteWeight { + // Deliberately not marked as known. Known says the bounds hold + // evidence, and a suspicion is held apart from them precisely + // because we cannot say that. A channel under suspicion is still + // priced off the prior, discounted at the amount named. + l.normalize(capacity) + + return + } + + // Convicted. The suspicion becomes an ordinary upper bound, and the + // quarantine that held it is emptied, since from here it is the bound + // that speaks. + suspect := l.SuspectAmt + if l.UpperFail == 0 || suspect < l.UpperFail { + l.UpperFail = suspect + } + + failed := suspect * intervalSuspectEstimateNumerator / + intervalSuspectEstimateDenominator + if l.Estimate == 0 || failed < l.Estimate { + l.Estimate = failed + } + + l.Confidence = math.Max(l.Confidence, intervalFailureConfidence) + l.Failures++ + l.Known = true + l.Restored = false + l.clearSuspect() + l.normalize(capacity) +} + +// suspectFactor returns the discount a quarantined observation applies to the +// given amount. It is always above zero: a suspicion we have not convicted must +// never say impossible, because an impossible amount is never attempted and the +// attempt is the only thing that could clear the suspicion. +func (l *LiquidityInterval) suspectFactor(amt lnwire.MilliSatoshi) float64 { + if l.SuspectAmt == 0 || amt < l.SuspectAmt { + return 1 + } + + weight := math.Min(l.SuspectWeight, intervalSuspectPenaltyCap) + + return math.Exp(-intervalSuspectPenalty * weight) +} + // intervalStrongObservation reports whether an observation of the given amount // is large enough to be allowed to move the mode latch. func intervalStrongObservation(amt, capacity lnwire.MilliSatoshi) bool { @@ -506,6 +677,11 @@ func (l *LiquidityInterval) Probability(amt, probability := l.rawProbability(amt, capacity, prior) + // A quarantined failure discounts the amount it named without ruling it + // out. Multiplying leaves a proven zero at zero and leaves a restored + // belief above its floor, so neither of those rules is disturbed. + probability *= l.suspectFactor(amt) + // A belief we restored from disk describes a network that has had every // chance to move on since we wrote it down. The bounds are still worth // something, which is why we keep them, but they are no longer allowed @@ -785,6 +961,13 @@ func (l *LiquidityInterval) recordSettlement(reverse *LiquidityInterval, l.UpperFail = 0 } + // This is the only place ProvenOK is ever written. The amount really + // moved over this channel in this direction, which is the one claim + // strong enough to clear a suspicion. + if amt > l.ProvenOK { + l.ProvenOK = amt + } + l.Known = true l.Restored = false l.Confidence = math.Max(l.Confidence, intervalSettleConfidence) diff --git a/routing/interval_config.go b/routing/interval_config.go index 8937c9ba8f0..d48336fe989 100644 --- a/routing/interval_config.go +++ b/routing/interval_config.go @@ -82,6 +82,18 @@ type IntervalConfig struct { // payment that still cannot be routed is given up on rather than cut // any finer. MinShardAmt lnwire.MilliSatoshi + + // DisableQuarantine stops the router holding an ambiguous failure as + // soft evidence against the channels that could have caused it. With it + // set, a failure that cannot name a hop leaves nothing behind in the + // node wide store and is handled entirely within the payment, which is + // what the router did before the quarantine existed. + // + // The sense is inverted so that the zero value keeps the behaviour the + // router was validated with. The mechanism measured as a null on the + // tiers built to reward it, so this exists to make turning it off a + // configuration change rather than a patch. + DisableQuarantine bool } // DefaultIntervalConfig returns the configuration the interval router was diff --git a/routing/interval_quarantine_test.go b/routing/interval_quarantine_test.go new file mode 100644 index 00000000000..1f3772c2871 --- /dev/null +++ b/routing/interval_quarantine_test.go @@ -0,0 +1,596 @@ +package routing + +import ( + "testing" + + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/routing/route" + "github.com/stretchr/testify/require" +) + +// TestIntervalQuarantinePricesSoftly tests the property that separates a +// quarantined observation from a bound. A failure we cannot attribute makes an +// amount less attractive and never makes it impossible, because an impossible +// amount is never attempted and an attempt is the only thing that could show +// the suspicion was misplaced. +func TestIntervalQuarantinePricesSoftly(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + store := NewIntervalStore(0) + clean := store.Probability(testIntervalKey, amt, capacity) + + // One ambiguous failure naming three channels, so a third of the blame + // lands here. + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0/3) + + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, amt, interval.SuspectAmt) + require.InDelta(t, 1.0/3, interval.SuspectWeight, 1e-9) + + // The bounds are untouched, which is the whole point of the quarantine. + require.Zero(t, interval.UpperFail) + + // The amount is discounted but still reachable. + suspected := store.Probability(testIntervalKey, amt, capacity) + require.Less(t, suspected, clean) + require.Greater(t, suspected, 0.0) + + // Only the amount the failure named, and larger ones, are discounted. A + // smaller amount is untouched, since nothing was said against it. + require.Equal( + t, store.Probability(testIntervalKey, amt/4, capacity), + NewIntervalStore(0).Probability(testIntervalKey, amt/4, capacity), + ) + require.Less( + t, store.Probability(testIntervalKey, amt*3/2, capacity), + clean, + ) + + // More agreement discounts harder, without ever reaching zero. + previous := suspected + for i := 0; i < 3; i++ { + store.RecordSuspectFailure( + testIntervalKey, amt, capacity, 1.0/3, + ) + + current := store.Probability(testIntervalKey, amt, capacity) + if store.Get(testIntervalKey, capacity).SuspectAmt == 0 { + // Promoted, which the next test covers. + break + } + + require.LessOrEqual(t, current, previous) + require.Greater(t, current, 0.0) + previous = current + } +} + +// TestIntervalQuarantinePromotes tests that agreement convicts. Enough +// independent failures naming the same channel and the same amount turn the +// suspicion into an ordinary upper bound, at which point it prices like any +// other thing we have watched fail. +func TestIntervalQuarantinePromotes(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + store := NewIntervalStore(0) + + // Failures naming two suspects each, so half the blame lands here every + // time. Three of them clear the promotion threshold. + for i := 0; i < 2; i++ { + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 0.5) + + require.NotZero( + t, store.Get(testIntervalKey, capacity).SuspectAmt, + "convicted on %d reports", i+1, + ) + require.Zero(t, store.Get(testIntervalKey, capacity).UpperFail) + } + + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.05) + + // Convicted. The suspicion is now a bound, and the quarantine that held + // it is empty, since from here the bound is what speaks. + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, amt, interval.UpperFail) + require.Zero(t, interval.SuspectAmt) + require.Zero(t, interval.SuspectWeight) + require.Less(t, interval.Estimate, amt) + + // And it prices as a bound does. + require.Zero(t, store.Probability(testIntervalKey, amt, capacity)) +} + +// TestIntervalQuarantinePromotesAtSmallestAmount tests that the quarantine +// keeps the tightest amount it has been shown, so that a conviction bounds the +// channel where the evidence actually put it. +func TestIntervalQuarantinePromotesAtSmallestAmount(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + + store := NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, capacity/2, capacity, 1.0) + store.RecordSuspectFailure(testIntervalKey, capacity/4, capacity, 1.0) + + require.Equal( + t, capacity/4, store.Get(testIntervalKey, capacity).SuspectAmt, + ) + + store.RecordSuspectFailure(testIntervalKey, capacity/2, capacity, 1.0) + + require.Equal( + t, capacity/4, store.Get(testIntervalKey, capacity).UpperFail, + ) +} + +// TestIntervalQuarantineClearsOnContradiction tests what does and does not +// count as proof that a suspicion was misplaced. +// +// Only a settlement counts. A lower bound is not enough, because a lower bound +// also rises when some hop reports a failure and we infer that the hops before +// it forwarded. That inference is sound exactly when the report names the right +// hop, and misattribution is the case where it does not. +func TestIntervalQuarantineClearsOnContradiction(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + // A probe does not clear a suspicion, however large. It is an inference + // from somebody else's failure report, and the report may have named + // the wrong hop. + store := NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordProbe(testIntervalKey, amt*3/2, capacity) + + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, amt, interval.SuspectAmt) + require.NotZero(t, interval.SuspectWeight) + require.Zero(t, interval.ProvenOK) + + // A settlement of the suspected amount does clear it. The money moved, + // which is the one thing no misattribution can manufacture. + store = NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordSettlement(testIntervalKey, amt, capacity) + + interval = store.Get(testIntervalKey, capacity) + require.Zero(t, interval.SuspectAmt) + require.Zero(t, interval.SuspectWeight) + require.Equal(t, amt, interval.ProvenOK) + + // So does a settlement of more than the suspected amount. + store = NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, amt/2, capacity, 1.0) + store.RecordSettlement(testIntervalKey, amt, capacity) + require.Zero(t, store.Get(testIntervalKey, capacity).SuspectAmt) + + // A settlement of less does not, since it says nothing about the amount + // the failure named. + store = NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordSettlement(testIntervalKey, amt/4, capacity) + require.Equal( + t, amt, store.Get(testIntervalKey, capacity).SuspectAmt, + ) + + // A suspicion about an amount we have watched settle is never held in + // the first place. + store = NewIntervalStore(0) + store.RecordSettlement(testIntervalKey, amt, capacity) + store.RecordSuspectFailure(testIntervalKey, amt/2, capacity, 1.0) + require.Zero(t, store.Get(testIntervalKey, capacity).SuspectAmt) + + // Proof is monotone: a later, smaller settlement does not walk it back + // and re-arm a suspicion the larger one had cleared. + store.RecordSettlement(testIntervalKey, amt/8, capacity) + require.Equal(t, amt, store.Get(testIntervalKey, capacity).ProvenOK) +} + +// TestIntervalQuarantineSurvivesMisattribution tests the failure shape this +// trust boundary exists for. +// +// A failure reported by some hop makes us write a lower bound on every hop +// before it, because forwarding is what got the payment that far. Under +// attribution shift the report names a hop downstream of the one that actually +// refused, which puts the guilty channel before the reported index and hands it +// a lower bound claiming it carried the very amount it just turned down. If +// that bound counted as proof of innocence, the culprit would be struck off +// every suspect list it belonged on, and the weight of the failure would +// concentrate on the innocent channels that remained. +func TestIntervalQuarantineSurvivesMisattribution(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + store := NewIntervalStore(0) + + // The shifted report: the true culprit is named as a hop that forwarded, + // so it collects a lower bound for the amount it actually refused. + store.RecordProbe(testIntervalKey, amt, capacity) + require.Equal(t, amt, store.Get(testIntervalKey, capacity).LowerOK) + require.Zero(t, store.Get(testIntervalKey, capacity).ProvenOK) + + // A later ambiguous failure names the same channel at the same amount. + // The false bound must not suppress it. + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, amt, interval.SuspectAmt, + "a probe derived bound suppressed a suspicion") + require.NotZero(t, interval.SuspectWeight) + + // The suspicion prices, so the channel is discounted at the amount it + // keeps being blamed for. + clean := NewIntervalStore(0) + clean.RecordProbe(testIntervalKey, amt, capacity) + require.Less( + t, store.Probability(testIntervalKey, amt, capacity), + clean.Probability(testIntervalKey, amt, capacity), + ) + + // Corroboration still convicts. The false bound sits below the amount + // the ambiguous failures name, which is the ordinary case, and the + // promotion writes the bound it should. + store = NewIntervalStore(0) + store.RecordProbe(testIntervalKey, amt/2, capacity) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.05) + + interval = store.Get(testIntervalKey, capacity) + require.Equal(t, amt, interval.UpperFail) + require.Zero(t, interval.SuspectAmt) + + // One case is worth pinning because it is left deliberately alone. When + // a false bound lands at exactly the amount the failures name, the + // promotion is written and then dropped again by the rule that a lower + // bound and an upper bound at the same amount cannot both stand. That + // rule is ordinary bound maintenance, it is not part of the quarantine, + // and rewriting it would be a change nobody has measured. The suspicion + // is still held and still priced up to that point, which is the part + // that matters. + store = NewIntervalStore(0) + store.RecordProbe(testIntervalKey, amt, capacity) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.05) + + require.Zero(t, store.Get(testIntervalKey, capacity).UpperFail) + + // Ground truth still speaks. A settlement over the same channel clears + // a suspicion that a hundred probes could not. + store = NewIntervalStore(0) + store.RecordProbe(testIntervalKey, amt, capacity) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + require.NotZero(t, store.Get(testIntervalKey, capacity).SuspectAmt) + + store.RecordSettlement(testIntervalKey, amt, capacity) + require.Zero(t, store.Get(testIntervalKey, capacity).SuspectAmt) +} + +// TestIntervalQuarantineSuspectListIgnoresProbes tests the same boundary at the +// place the session applies it: a hop is struck off the suspect list of an +// unattributable failure only when a settlement has proven it, never when a +// probe has merely implied it. +func TestIntervalQuarantineSuspectListIgnoresProbes(t *testing.T) { + t.Parallel() + + capacity := lnwire.NewMSatFromSatoshis(budgetCapacity) + amt := lnwire.MilliSatoshi(600_000_000) + + // A route with two hops that are not ours, so an unattributable failure + // over it has two suspects. + rt := &route.Route{ + TotalAmount: amt, + SourcePubKey: createPubkey(sourceNodeID), + Hops: []*route.Hop{ + { + PubKeyBytes: createPubkey(firstRelayID), + ChannelID: 1, + AmtToForward: amt, + }, + { + PubKeyBytes: createPubkey(secondRelayID), + ChannelID: 9, + AmtToForward: amt, + }, + { + PubKeyBytes: createPubkey(targetNodeID), + ChannelID: 4, + AmtToForward: amt, + }, + }, + } + + suspects := []IntervalKey{ + { + ChanID: 9, + From: createPubkey(firstRelayID), + To: createPubkey(secondRelayID), + }, + { + ChanID: 4, + From: createPubkey(secondRelayID), + To: createPubkey(targetNodeID), + }, + } + + report := func(prove bool) *IntervalStore { + session, store := newCorridorSession( + t, lnwire.NewMSatFromSatoshis(600_000), 1, + ) + for _, key := range intervalRouteKeys(rt) { + session.capacities[key] = capacity + } + + // Both hops carry a probe derived lower bound covering the + // amount, which is what a shifted report leaves behind. + for _, key := range suspects { + store.RecordProbe(key, amt, capacity) + } + + // One of them additionally has a settlement behind it. + if prove { + store.RecordSettlement(suspects[0], amt, capacity) + } + + session.ReportAttemptFailure(0, rt, nil, nil) + + return store + } + + // With only probes behind them, both hops stay on the list and both + // take a share of the suspicion. + store := report(false) + for _, key := range suspects { + require.NotZero(t, store.Get(key, capacity).SuspectAmt, + "channel %v was struck off on a probe", key.ChanID) + } + + // With a settlement behind the first, it is struck off, and being the + // only suspect left makes the second a certainty by elimination rather + // than a suspicion. + store = report(true) + require.Zero(t, store.Get(suspects[0], capacity).SuspectAmt) + require.Zero(t, store.Get(suspects[1], capacity).SuspectAmt) + require.NotZero(t, store.Get(suspects[1], capacity).UpperFail) +} + +// TestIntervalQuarantineSubsumedByBound tests that a failure we do trust +// swallows a suspicion reaching for the same thing, so that the two do not +// discount the same amount twice over. +func TestIntervalQuarantineSubsumedByBound(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + amt := capacity / 2 + + store := NewIntervalStore(0) + store.RecordSuspectFailure(testIntervalKey, amt, capacity, 1.0) + store.RecordFailure(testIntervalKey, amt/2, capacity) + + interval := store.Get(testIntervalKey, capacity) + require.Equal(t, amt/2, interval.UpperFail) + require.Zero(t, interval.SuspectAmt) +} + +// TestIntervalQuarantineWritesOneDirection tests that an ambiguous failure says +// nothing about the other side of the channel. The inference that liquidity +// missing here is liquidity present there only holds when we know the failure +// happened here. +func TestIntervalQuarantineWritesOneDirection(t *testing.T) { + t.Parallel() + + capacity := testIntervalCapacity + + store := NewIntervalStore(0) + store.RecordSuspectFailure( + testIntervalKey, capacity/2, capacity, 1.0, + ) + + reverse := store.Get(testIntervalKey.Reverse(), capacity) + require.False(t, reverse.Known) + require.Zero(t, reverse.LowerOK) + require.Zero(t, reverse.SuspectAmt) +} + +// TestIntervalQuarantineIgnoresUninformative tests that a quarantine entry is +// only made when there is something to record. +func TestIntervalQuarantineIgnoresUninformative(t *testing.T) { + t.Parallel() + + store := NewIntervalStore(0) + + store.RecordSuspectFailure(testIntervalKey, 0, testIntervalCapacity, 1) + store.RecordSuspectFailure(testIntervalKey, 100, 0, 1) + store.RecordSuspectFailure(testIntervalKey, 100, testIntervalCapacity, 0) + + require.Zero(t, store.Len()) +} + +// TestIntervalSessionQuarantinesAmbiguousFailure tests the path a real payment +// takes. A failure nobody claims, over a route with several plausible culprits, +// leaves a discount on each of them and a bound on none. +func TestIntervalSessionQuarantinesAmbiguousFailure(t *testing.T) { + t.Parallel() + + const capacitySat = 100_000 + + graph := newIntervalTestGraph(t, []byte{firstRelayID}, capacitySat) + amt := lnwire.NewMSatFromSatoshis(40_000) + ctx := newIntervalTestCtx(t, graph, amt, 1, false) + + rt, err := ctx.session.RequestRoute( + amt, lnwire.MaxMilliSatoshi, 0, 0, nil, + ) + require.NoError(t, err) + require.Len(t, rt.Hops, 2) + + // A failure with no source and no message, which is what an unreadable + // onion error looks like by the time it reaches us. + ctx.session.ReportAttemptFailure(0, rt, nil, nil) + + // Our own first hop is never a suspect, so the only channel that could + // be blamed is the interior one, and a single suspect is an elimination + // rather than a guess: that one is bounded outright. + interior := IntervalKey{ + ChanID: 2, + From: createPubkey(firstRelayID), + To: createPubkey(targetNodeID), + } + capacity := lnwire.NewMSatFromSatoshis(capacitySat) + require.NotZero(t, ctx.store.Get(interior, capacity).UpperFail) + + // Now the ambiguous case, over a route with two channels neither of + // which is ours. + session, store := newCorridorSession( + t, lnwire.NewMSatFromSatoshis(600_000), 1, + ) + longRoute := &route.Route{ + TotalAmount: 600_000_000, + SourcePubKey: createPubkey(sourceNodeID), + Hops: []*route.Hop{ + { + PubKeyBytes: createPubkey(firstRelayID), + ChannelID: 1, + AmtToForward: 600_000_000, + }, + { + PubKeyBytes: createPubkey(secondRelayID), + ChannelID: 9, + AmtToForward: 600_000_000, + }, + { + PubKeyBytes: createPubkey(targetNodeID), + ChannelID: 4, + AmtToForward: 600_000_000, + }, + }, + } + + // The session has to know the capacities to record anything, which it + // normally learns while path finding. + capacity = lnwire.NewMSatFromSatoshis(budgetCapacity) + for _, key := range intervalRouteKeys(longRoute) { + session.capacities[key] = capacity + } + + session.ReportAttemptFailure(0, longRoute, nil, nil) + + // Two suspects, so each carries a quarantined discount and neither + // carries a bound. + suspects := []IntervalKey{ + { + ChanID: 9, + From: createPubkey(firstRelayID), + To: createPubkey(secondRelayID), + }, + { + ChanID: 4, + From: createPubkey(secondRelayID), + To: createPubkey(targetNodeID), + }, + } + + for _, key := range suspects { + interval := store.Get(key, capacity) + + require.NotZero(t, interval.SuspectAmt, "no suspicion on %v", + key.ChanID) + require.Zero(t, interval.UpperFail, "bound placed on %v by an "+ + "ambiguous failure", key.ChanID) + require.Greater( + t, store.Probability(key, 600_000_000, capacity), 0.0, + ) + } +} + +// TestIntervalQuarantineSeverable tests that the quarantine can be switched off +// without touching anything else. It measured as a null on the tiers built to +// reward it, so whether it ships is a decision somebody should be able to make +// with a config field rather than a patch. +func TestIntervalQuarantineSeverable(t *testing.T) { + t.Parallel() + + // The zero value keeps the mechanism on, which is the behaviour every + // published measurement of this router was taken with. + require.False(t, IntervalConfig{}.DisableQuarantine) + require.False(t, DefaultIntervalConfig().DisableQuarantine) + + route := func(disabled bool) (*IntervalStore, []IntervalKey) { + session, store := newCorridorSession( + t, lnwire.NewMSatFromSatoshis(600_000), 1, + ) + session.cfg.DisableQuarantine = disabled + + // A route with two hops that are not ours, so an unattributable + // failure over it has two suspects and neither can be named. + rt := &route.Route{ + TotalAmount: 600_000_000, + SourcePubKey: createPubkey(sourceNodeID), + Hops: []*route.Hop{ + { + PubKeyBytes: createPubkey(firstRelayID), + ChannelID: 1, + AmtToForward: 600_000_000, + }, + { + PubKeyBytes: createPubkey( + secondRelayID, + ), + ChannelID: 9, + AmtToForward: 600_000_000, + }, + { + PubKeyBytes: createPubkey(targetNodeID), + ChannelID: 4, + AmtToForward: 600_000_000, + }, + }, + } + + keys := intervalRouteKeys(rt) + for _, key := range keys { + session.capacities[key] = lnwire.NewMSatFromSatoshis( + budgetCapacity, + ) + } + + session.ReportAttemptFailure(0, rt, nil, nil) + + return store, keys + } + + capacity := lnwire.NewMSatFromSatoshis(budgetCapacity) + + // On, the suspects carry a discount. + store, keys := route(false) + + var suspected int + for _, key := range keys { + if store.Get(key, capacity).SuspectAmt != 0 { + suspected++ + } + } + require.NotZero(t, suspected) + + // Off, the store hears nothing at all. Nothing is recorded, so nothing + // prices, and the payment falls back to handling the failure with the + // penalties that live and die with it. + store, keys = route(true) + + require.Zero(t, store.Len()) + for _, key := range keys { + interval := store.Get(key, capacity) + + require.Zero(t, interval.SuspectAmt) + require.Zero(t, interval.SuspectWeight) + require.False(t, interval.Known) + } +} diff --git a/routing/interval_session.go b/routing/interval_session.go index e689794ff21..51ee01e2b87 100644 --- a/routing/interval_session.go +++ b/routing/interval_session.go @@ -985,10 +985,14 @@ func (p *intervalPaymentSession) recordUnattributedFailure(rt *route.Route, continue } - // A hop we have already proven carries this amount cannot be - // the one that refused it. + // A hop we have watched settle this amount is struck off the + // list. Only a settlement counts: a lower bound can also come + // from a failure reported further along the route, and when the + // report names the wrong hop that bound lands on the very + // channel that refused. See LiquidityInterval.ProvenOK. amt := intervalHopAmount(rt, i) - if p.store.Get(key, p.capacities[key]).LowerOK >= amt { + proven := p.store.Get(key, p.capacities[key]).ProvenOK + if proven != 0 && proven >= amt { continue } @@ -1013,8 +1017,30 @@ func (p *intervalPaymentSession) recordUnattributedFailure(rt *route.Route, return } + // How much this one failure implicates any one of its suspects falls + // with the number of them, which is also the weight the quarantine + // records. Three failures naming two channels each will convict a + // channel they agree on; five are needed when each names five. + weight := 1 / math.Sqrt(float64(len(suspects))) + share := intervalSuspicionMass / math.Sqrt(float64(len(suspects))) for _, item := range suspects { + // Hold the observation in the store's quarantine, where it + // prices as a discount for every payment rather than only for + // this one, and where enough agreement across payments turns it + // into a bound. Until then it is not allowed to rule anything + // out, because we cannot say it happened here. + // + // This is the only place anything is ever written to the + // quarantine, so switching it off here leaves the whole + // mechanism inert: nothing is recorded, so nothing prices. + if !p.cfg.DisableQuarantine { + p.store.RecordSuspectFailure( + item.key, item.amt, + p.capacities[item.key], weight, + ) + } + p.suspects[item.key]++ p.penalties[item.key] += share diff --git a/routing/interval_store.go b/routing/interval_store.go index 2c82ee175e3..ec074523ff9 100644 --- a/routing/interval_store.go +++ b/routing/interval_store.go @@ -349,6 +349,43 @@ func (s *IntervalStore) RecordSettlement(key IntervalKey, }) } +// RecordSuspectFailure quarantines a failure that could not be attributed to +// one channel with confidence. The weight says how much this failure implicates +// this channel rather than the others it could equally have been. +// +// Unlike the three observations above, this one writes only the forward +// direction. A failure we are not sure happened here is not evidence about the +// liquidity on the other side of the channel, and inferring one from the other +// is only sound when we know where the failure was. +func (s *IntervalStore) RecordSuspectFailure(key IntervalKey, + amt, capacity lnwire.MilliSatoshi, weight float64) { + + if amt == 0 || capacity == 0 || weight <= 0 { + return + } + + if amt > capacity { + amt = capacity + } + + s.mu.Lock() + defer s.mu.Unlock() + + entry := s.entryLocked(key) + entry.recordSuspect(amt, capacity, weight) + + // The quarantine itself is never written down. It is soft evidence about + // an attribution we did not trust, and carrying it across a restart + // would mean restoring a suspicion that nothing since has been able to + // clear. Only a promotion, which leaves an ordinary bound behind, is + // worth persisting. + if entry.SuspectAmt == 0 { + s.markDirtyLocked(key) + } + + s.evictLocked() +} + // update applies an observation to both directions of a channel under the // store's lock. Observations of a zero amount, or of a channel whose capacity // we do not know, carry no information the model can use and are dropped. The From b1e95f71a047a2113b4162087453dc55e28c51f8 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:20:29 -0700 Subject: [PATCH 8/9] itest: cover the interval router over a three hop network In this commit, we exercise the interval router against real nodes. Only the sender runs it, so the test also covers routing through peers that do not, which is the situation any node turning this on would be in. The payment worth testing is the second one. We drain the middle hop, watch a large payment fail there, and then send a small one without resetting anything and without waiting. The stock router would have penalized the node pair on that failure and would need the penalty to decay before it would offer the pair again. The interval router recorded an amount rather than a verdict, so every smaller amount over the same channel stayed routable and the recovery costs one attempt. That is the behaviour the whole design is for, and the one thing about it an integration test can show that a unit test cannot. --- itest/list_on_test.go | 4 ++ itest/lnd_interval_router_test.go | 110 ++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 itest/lnd_interval_router_test.go diff --git a/itest/list_on_test.go b/itest/list_on_test.go index 1301a266e30..c7efacd69fb 100644 --- a/itest/list_on_test.go +++ b/itest/list_on_test.go @@ -278,6 +278,10 @@ var allTestCases = []*lntest.TestCase{ Name: "multi-hop payments", TestFunc: testMultiHopPayments, }, + { + Name: "interval router multi hop payment", + TestFunc: testIntervalRouterMultiHopPayment, + }, { Name: "estimate route fee", TestFunc: testEstimateRouteFee, diff --git a/itest/lnd_interval_router_test.go b/itest/lnd_interval_router_test.go new file mode 100644 index 00000000000..1108efa4fe2 --- /dev/null +++ b/itest/lnd_interval_router_test.go @@ -0,0 +1,110 @@ +package itest + +import ( + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/lightningnetwork/lnd/lnrpc" + "github.com/lightningnetwork/lnd/lnrpc/routerrpc" + "github.com/lightningnetwork/lnd/lntest" + "github.com/stretchr/testify/require" +) + +// testIntervalRouterMultiHopPayment tests the experimental interval router end +// to end over a three hop network, with only the sender running it so that the +// test also covers a node routing for peers that do not. +// +// The payment the router is interesting on is the second one. We drain the +// middle hop so that a large payment cannot get through, watch it fail at that +// hop, and then send a small one without resetting anything. The stock router +// would have penalized the node pair and would need the penalty to decay; the +// interval router records the amount that failed as a bound, which leaves every +// smaller amount over the same channel perfectly routable. The retry is +// therefore a test of what the router remembered, not of what it forgot. +func testIntervalRouterMultiHopPayment(ht *lntest.HarnessTest) { + const ( + chanAmt = btcutil.Amount(300_000) + + // smallPayment fits comfortably inside what is left of the + // middle hop after the drain below. + smallPayment = btcutil.Amount(10_000) + + // largePayment is well above it. + largePayment = btcutil.Amount(100_000) + + // drainPayment leaves the middle hop with roughly 50k of + // outbound liquidity, less its reserve and fee buffer. + drainPayment = btcutil.Amount(250_000) + ) + + // Build Alice -> Bob -> Carol, with only Alice routing on intervals. + cfgs := [][]string{{"--routerrpc.router=interval"}, nil, nil} + _, nodes := ht.CreateSimpleNetwork( + cfgs, lntest.OpenChannelParams{Amt: chanAmt}, + ) + alice, bob, carol := nodes[0], nodes[1], nodes[2] + + // sendPayment is a helper that pays a fresh invoice of the given amount + // from Alice to Carol over a single HTLC, so that the outcome is about + // route choice rather than about how the payment was split. + sendPayment := func(amt btcutil.Amount) *routerrpc.SendPaymentRequest { + invoice := carol.RPC.AddInvoice(&lnrpc.Invoice{ + Value: int64(amt), + }) + + return &routerrpc.SendPaymentRequest{ + PaymentRequest: invoice.PaymentRequest, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + MaxParts: 1, + } + } + + // With liquidity everywhere, the interval router finds the two hop + // route and the payment settles. + payment := ht.SendPaymentAssertSettled(alice, sendPayment(smallPayment)) + require.Len(ht, payment.Htlcs, 1) + require.Len(ht, payment.Htlcs[0].Route.Hops, 2) + require.Equal( + ht, carol.PubKeyStr, + payment.Htlcs[0].Route.Hops[1].PubKey, + ) + + // Now drain the middle hop by having Bob pay Carol, which leaves Bob + // without the outbound liquidity to forward a large payment onwards. + drainInvoice := carol.RPC.AddInvoice(&lnrpc.Invoice{ + Value: int64(drainPayment), + }) + ht.SendPaymentAssertSettled(bob, &routerrpc.SendPaymentRequest{ + PaymentRequest: drainInvoice.PaymentRequest, + TimeoutSeconds: 60, + FeeLimitMsat: noFeeLimitMsat, + MaxParts: 1, + }) + + // Alice cannot know that from the graph, so she tries the large payment + // and Bob refuses to forward it. With no second route to Carol and no + // room to split, the payment fails. + ht.SendPaymentAssertFail( + alice, sendPayment(largePayment), + lnrpc.PaymentFailureReason_FAILURE_REASON_NO_ROUTE, + ) + ht.AssertLastHTLCError(alice, lnrpc.Failure_TEMPORARY_CHANNEL_FAILURE) + + // The failure told Alice's router an amount, not a verdict on the + // channel. Nothing is reset here, and no penalty is waited out: a + // smaller payment over the very same hop settles immediately, because + // the bound the router recorded rules out the amount that failed and + // says nothing against this one. + payment = ht.SendPaymentAssertSettled(alice, sendPayment(smallPayment)) + require.Len(ht, payment.Htlcs, 1) + require.Len(ht, payment.Htlcs[0].Route.Hops, 2) + require.Equal( + ht, bob.PubKeyStr, payment.Htlcs[0].Route.Hops[0].PubKey, + ) + + // The whole of that recovery took one attempt, since the router asked + // for an amount it had no evidence against rather than retrying the one + // it had just watched fail. + require.Equal( + ht, lnrpc.HTLCAttempt_SUCCEEDED, payment.Htlcs[0].Status, + ) +} From e0ced4027661d5a8cb40dd2894cec61adcfc8b78 Mon Sep 17 00:00:00 2001 From: Olaoluwa Osuntokun Date: Fri, 7 Aug 2026 18:20:51 -0700 Subject: [PATCH 9/9] docs: explain the interval router In this commit, we write down what the router does and why, for a contributor who knows lnd and has never seen this line of work. The algorithm inverts the question path finding asks: rather than testing whether the graph can carry an amount fixed before the search began, it asks what the network will accept and picks the amount and the route together. Everything else follows from that, so the document leads with it. The rest covers the belief itself, the bimodal prior and why its scale is a fraction of capacity rather than a number of satoshis, the label setting search and why a node has to keep more than one answer, how the shard ladder is priced and where its exchange rate between fees and reliability comes from, the trust boundary the quarantine draws, and how all of it coexists with mission control, which keeps running untouched. The limitations section is the part worth reading twice. It states the blinded path fallback and what finishing it would take, why a bound restored from disk is deliberately weaker than one we just watched hold, why an ambiguous failure is recorded against a node pair rather than a channel, what the search costs against a single Dijkstra run, and that the constants in the model were selected by a search rather than derived, with some of them surely fitted to the simulator that produced them. The release notes gain an entry under functional enhancements and one under database, since the router brings a table with it. --- docs/interval_routing.md | 427 +++++++++++++++++++++ docs/release-notes/release-notes-0.22.0.md | 20 + 2 files changed, 447 insertions(+) create mode 100644 docs/interval_routing.md diff --git a/docs/interval_routing.md b/docs/interval_routing.md new file mode 100644 index 00000000000..b3374865475 --- /dev/null +++ b/docs/interval_routing.md @@ -0,0 +1,427 @@ +# The Interval Router + +What is the largest amount this route can still carry? + +That question is the whole of the difference between the interval router and +the one lnd has always shipped. The stock router asks whether the graph can +carry an amount that was fixed before path finding began, and when the answer +is no, it halves the amount and asks again. The interval router asks what the +network will accept, and picks the amount and the route together. + +This is written for an lnd contributor seeing the algorithm for the first +time. It is off by default. + +## What the router remembers + +Mission control remembers a penalty. When an attempt fails at some node pair, +it records that pair as a bad bet and lets the record fade on a half life, so +that a channel which was empty an hour ago becomes worth trying again today. + +The interval router remembers an amount range instead. For each direction of +each channel it keeps three numbers: + +- `LowerOK`, the largest amount it has watched pass. Anything at or below this + is treated as near certain. +- `UpperFail`, the smallest amount it has watched fail. Anything at or above + this is treated as impossible. +- `Estimate`, its best guess at the balance, somewhere between the two. + +Alongside them it keeps a confidence, which rises as evidence accumulates, and +a classification: whether the channel looks nearly empty in this direction, +nearly full, or neither. + +There is no clock anywhere in the model. A bound moves when evidence arrives +and never because time has passed. That is the largest departure from mission +control, and the one that takes the most care to get right, since a belief with +no expiry is a belief that has to be correct. + +### Every observation writes both directions + +A payment attempt teaches the router three kinds of thing, and each of them +writes the channel it names and also the same channel in reverse. + +A **failure** at some hop drops that direction's `UpperFail` to the amount that +was refused. It also raises the reverse direction's `LowerOK`, because +liquidity that is not on this side of the funding output is on the other side. +A channel that cannot send you 400,000 satoshis is a channel that can probably +send 400,000 satoshis back. Mission control makes no such inference; its +two directions of a pair are wholly independent records. + +A **probe** is what the router learns about the hops it did not fail at. If an +attempt fails four hops out, the first three hops forwarded, which proves they +could carry what they were handed. That raises their `LowerOK` and lowers the +reverse `UpperFail`. + +A **settlement** is different in kind from both, because the money actually +moved. The forward interval slides down by the amount that left and the reverse +interval slides up by the same, so the router's picture of the channel tracks +the payment it just made rather than merely narrowing around it. + +### Evidence it does not trust yet + +Not every failure says where it happened. An unreadable error, or one no node +claims, leaves several hops on the route that could each have been the one to +refuse. Writing an upper bound on all of them would be a claim of certainty +about channels that may be perfectly healthy, and this model has no way back +from a bound: an amount it calls impossible is never attempted, so the attempt +that would clear the mistake never happens. + +Such an observation goes into a quarantine instead, held per directed channel +apart from the bounds. It records the smallest amount an ambiguous failure has +named and how much corroboration stands behind it, where a failure naming two +suspects contributes half as much as one naming a single suspect. Quarantined +evidence prices as a discount on the amount it named and never as an +impossibility. Once enough independent failures agree on the same channel it is +promoted into an ordinary upper bound. + +Only a settlement clears a suspicion. That is a narrower rule than it sounds, +and it is the one thing in this section worth understanding. When a hop reports +a failure, the router writes a lower bound on every hop before it, because +forwarding is what carried the payment that far. The inference holds when the +report names the right hop. When blame arrives shifted downstream, which is one +of the ways a real network lies, the guilty channel sits before the reported +index and collects a lower bound saying it carried the amount it had just +refused. A quarantine that accepted lower bounds as proof of innocence would let +that channel out of every suspicion it belonged in, and would pile the blame +onto its innocent neighbours instead. So the router keeps a separate record of +what it has watched actually move, written by nothing but a settlement, and only +that clears a suspicion. The lower bound keeps every other job it has. + +### What it believes with no evidence at all + +Before any of that, the router needs an opinion about a channel it has never +touched. It assumes liquidity is bimodal: a channel is usually sitting near one +end of its range rather than politely balanced in the middle. So a small amount +is nearly certain to pass, an amount near the whole capacity is nearly certain +to fail, and the transition between the two is narrow. + +The width of that transition is a fraction of capacity, not a number of +satoshis. lnd's bimodal estimator takes a scale in millisatoshis, defaulting to +300,000 satoshis, which is 30% of a 1,000,000 satoshi channel and under 2% of a +16,000,000 satoshi one. Expressing the same shape as a percentage of capacity +is what lets one set of constants work on channels that differ in size by +orders of magnitude. + +## How it finds a route + +The search runs backwards from the destination, the same as `findPath`, and it +reuses the machinery that makes that walk correct: the edge unifier that picks +a policy per node pair, the bandwidth hints that speak for our own channels, +the fee and time lock limits, the onion payload budget, and the feature +validation of every node on the way. + +Two things differ. + +**The cost of a hop is additive.** The stock router minimizes fee plus a time +lock penalty, divided through by the probability of the route. The interval +router minimizes the negative logarithm of that probability, plus terms for +fee, for depth, and for how much of a channel the payment would fill. +Logarithms turn the product of hop probabilities into a sum, which is what +makes the search below tractable, and it is far gentler on a route the model is +merely unsure about than dividing by a small number is. + +**A node keeps several answers, not one.** Dijkstra keeps the single best +distance per node. This search keeps a bounded set of labels that no other +label beats on all three of cost, amount, and hop count at once. It needs to, +because the search runs backwards and fees accumulate as it goes. A route that +is cheaper but carries a larger amount is not comparable to one that is dearer +and carries less, since the larger amount may be refused further upstream. A +single distance per node cannot express that, and the amounts the shard ladder +wants to compare are exactly the ones where it matters. + +## How it splits a payment + +lnd splits a multi-path payment reactively. `paymentSession.RequestRoute` asks +for the whole remaining amount, and when path finding comes back empty it +halves the amount and tries again, stopping at the minimum shard size or the +part limit. Every split is a response to a failure, and every shard is a +power-of-two fraction. + +The interval router plans the split. For one call it builds a ladder of +candidate amounts, finds a route for each, and keeps the pairing of amount and +route with the best score. The ladder draws on four sources: + +1. the whole remaining amount, and the smallest shard that could still finish + the payment inside the parts left; +2. the amounts this payment has already proven do not fit, divided down until + they do; +3. even divisions of the remaining amount; +4. the halving chain, and small multiples of the smallest usable shard. + +Source 2 is the one that makes this more than a reordering of lnd's loop. A +failure at 400,000 satoshis, for instance, immediately puts 199,999 and 99,999 +into play, and the halving chain would reach those amounts only by accident. +Because every rung costs a full search, the ladder is capped, and the sources +are enumerated in the order above so that the cap keeps the informative rungs. + +Scoring a rung trades the risk of its route against how much of the payment it +would carry, with an appetite for large shards that responds to how the payment +is going: bolder once a part has settled and the payment is committed, more +cautious after several failures. + +One shard can get in the next one's way, so the router counts what it is +already holding. If a shard in flight is holding 200,000 satoshis on some +interior channel, a second shard of 300,000 needs that channel to have had room +for 500,000 when the router last looked at it, and that is the amount the model +is asked about. Folding the hold into the amount is the whole adjustment, +because every bound the model keeps already answers the question "was there +this much here". The holds are shared across the node, so a second payment +steps around the corridor a first payment is using instead of learning about +the contention by paying for a failed attempt. Our own channels are left out, +since the switch already subtracts in-flight HTLCs from the bandwidth it +reports for them. + +What the router will pay for reliability comes from the budget. The score being +minimized is in nats, so a fee has to be converted into one before it can be +compared against a probability, and the rate of that conversion is the only +thing standing between the search and a fee limit. The routers this design came +from set the rate as a fraction of the amount, which reads as a willingness to +pay a fifth of the payment to raise a route's probability by a factor of e. No +budget anybody would set is near that, so the fee term never bound and those +routers walked into limits they could not see. Here the remaining budget sets +the rate instead: a payment with 10,000 millisatoshis left will pay 5,000 of +them for one nat, so it declines expensive reliability on its own rather than +finding out when the route is refused. The rate is absolute, which means it +tightens in relative terms as the payment grows, and that is the direction a +budget quoted in parts per million needs. A payment with no budget keeps the +old amount relative rate, since there is nothing else to derive one from. + +Two smaller rules follow from the same concern. A node always keeps the +cheapest of its labels whatever that label's score, so a payment that cannot +afford the reliable routes still finds one it can. And no route is ever handed +out whose fee exceeds what is left of the budget. + +The payment lifecycle is untouched by any of this. It still asks for one route +at a time and dispatches one HTLC, or hash time locked contract, at a time. The +shard size simply rides back on the route, since `registerAttempt` already +reads `ReceiverAmt()` to decide whether a shard is the last one. + +## A payment, end to end + +The pieces above are easiest to see moving together. Suppose a node with the +router enabled sends 500,000 satoshis to a destination four hops away, over +channels it has never used, and suppose the true bottleneck is the third hop, +which holds 250,000. + +The first route request builds the shard ladder. Nothing has failed yet, so +the ladder's informative rungs are the whole amount and a handful of +divisions; the whole amount scores best, and the search returns a four hop +route for 500,000. The attempt fails at hop three, and the failure names it. + +Three things are written before the next request. Hop three's forward +direction takes `UpperFail = 500,000`: not a penalty, a ceiling. Hops one and +two forwarded, so their forward directions take `LowerOK = 500,000` and their +reverse directions learn the complementary bound. And hop three's reverse +direction takes `LowerOK = 500,000`, because the liquidity that was not there +to send is there to send back. + +The second route request now behaves differently in two ways at once. The +ladder contains 499,999 divided down: 249,999 and 166,666 are rungs, put +there directly by the failure, not reached by halving. And the search still +considers hop three for those smaller amounts, because a 250,000 shard sits +below no bound the model holds; the stock router would be routing around that +channel entirely, at every amount, for the length of a half-life. Say the +249,999 rung wins with the same route. It succeeds, the destination still +needs the rest, and the settlement slides hop three's forward interval down by +what just moved through it. + +The remaining 250,001 goes out on the next request the same way, over that +route or a better one, and the payment completes with three attempts. Every +bound written along the way outlives the payment: the next payment through +this corridor starts from what this one learned, and what it learned survives +a restart on nodes running the native SQL backend, in the clamped form the +limitations section describes. + +The same trace under the stock router reads differently at each step: the +failure penalizes the pair both ways, the retry halves 500,000 to 250,000 by +schedule rather than by evidence, and whether hop three is even considered +again depends on how much of its penalty has decayed, which is a function of +wall-clock time and not of anything the network said. + +## Living alongside mission control + +Turning the interval router on does not turn mission control off. Mission +control keeps running, keeps its history, keeps answering +`QueryProbability` and the rest of its remote procedure calls, and keeps +deciding whether a given failure is terminal for the payment. Only the choice +of route changes. + +Every attempt outcome therefore reaches two places. The payment lifecycle +reports it to mission control exactly as before, and then offers it to the +payment session through a small optional interface, `PaymentResultReporter`. +The stock session does not implement that interface, so with the flag off the +type assertion fails and nothing changes. + +The interval beliefs live in an `IntervalStore`, one per node and shared by +every payment, so what one payment learns is there for the next. On +a node running the native SQL backend the store also writes its beliefs down +and reads them back at startup. Elsewhere it is memory only and the router +starts cold after a restart. + +## Persistence and restarts + +The store holds at most 10,000 directed-channel entries in memory +(`DefaultMaxIntervalHistory`), evicting the least recently written when full, +so its footprint is bounded no matter how long the node runs or how large the +graph grows. + +On a node running the native SQL backend, the store also writes its beliefs +down. Writes are batched: a dirty entry waits at most the flush interval, +one second by default, before it reaches the `liquidity_intervals` table the +branch's migration adds. The interval is a config knob +(`routerrpc.intervalflushinterval`) because the right cadence is a judgment +about the node: a busy router may prefer a longer interval to cut write +amplification, and the cost of a longer interval is only the beliefs learned +in the final unflushed seconds before an unclean shutdown. + +At startup the store reads the table back and clamps everything it finds, as +the limitations section describes: restored bounds say likely and unlikely +rather than proven and impossible, and confidence is halved. Two kinds of +in-memory evidence are deliberately never written down. The quarantine is +not, because a suspicion restored from disk is one that nothing since could +have cleared. The settlement record that clears suspicions is not, because a +settlement from before a restart should not vouch for a channel today. Both +rules are the same instinct: fresh evidence outranks stored evidence, and +stored evidence never gets to overrule a live observation. + +On the kv backends there is no table to write to, so the router simply starts +cold after a restart. Nothing else changes. + +## Turning it on + +``` +[routerrpc] +routerrpc.router=interval +``` + +The other value is `default`, the stock stack, and it is what a node that says +nothing gets. With the flag off, none of the code described here is even +constructed: the server builds the same session source it always did, and the +result-reporting seam is a type assertion the stock session does not satisfy. + +The full config surface: + +| Option | Default | What it does | +|---|---|---| +| `routerrpc.router` | `default` | selects the routing engine | +| `routerrpc.intervalflushinterval` | `1s` | belief write-back cadence | + +`routerrpc.router=interval` enables everything this document describes. The +flush interval is how long a changed belief may wait before it is written to +the database, and it is only meaningful with the flag on and the native SQL +backend. + +One further switch lives in code rather than in the config file: +`IntervalConfig.DisableQuarantine` turns off the quarantine for ambiguous +failures while leaving every other mechanism in place. It exists so that the +one component validated only in simulation can be severed without touching +anything else. + +Turning the router off again is safe at any time. The stored beliefs remain +in the database and are simply not read; mission control's history was being +maintained the whole time, so the stock router resumes exactly where it would +have been. + +## Limitations + +**Payments to blinded paths fall back to the stock session.** Inside a blinded +path there is no channel for the model to key a belief on: the hops are opaque, +the intermediate amounts and expiries are deliberately zero, and an error from +inside the path arrives as `invalid_onion_blinding` from the introduction node, +which names nothing further in. Intervals on the visible prefix up to the +introduction node would work, since those hops are ordinary channels and a +failure past them proves they forwarded. Getting there also means teaching the +search about the dummy hop appended to blinded routes and about targeting the +nothing-up-my-sleeve (NUMS) key rather than the destination, so for now these +payments are handed to the router that already gets them right. + +**A restored bound is softer than a fresh one.** This model has no way back +from a wrong `UpperFail`: an amount the model calls impossible is never +attempted, and an attempt is the only thing that could correct the bound. That +is the right trade while the evidence is fresh and ours. It is a trapdoor for a +bound loaded from disk, because the network that bound describes has had every +restart and every rebalance since to move on. So a restored belief is clamped: +its upper bound says unlikely rather than impossible, its lower bound says +likely rather than proven, and its confidence is halved. The first fresh +observation clears all of it. Clamping is what makes keeping these beliefs +better than throwing them away at startup. + +**An ambiguous failure is recorded against the node pair.** The model wants to +key on a directed channel, because the quantity it tracks is the balance on one +side of one funding output. Under non-strict forwarding that is not always what +the evidence supports: a node asked to forward over one channel may use any +channel it has to the same peer, and the onion failure names neither. So when +the graph shows more than one channel between a pair, the observation is +written about the pair instead, at the granularity mission control has always +used. Pairs with a single channel, which is most of them, keep the full +resolution. + +**The quarantine is validated in simulation only.** Promotion after enough +agreement, and clearing on contradiction, come from a router bred against a +channel that lies about where failures happen. That router produced the flattest +degradation profile the work has measured, but it bought the flatness partly by +never giving up, and none of it has been measured on a real network. The +quarantine is held in memory only, as is the record of settlements that clears +it, since neither survives a restart with its meaning intact. Nor has it been +shown to be worth anything: on freshly generated files its effect on the +objective cannot be separated from zero in either direction, and it is kept +because the trust boundary it draws is the right one rather than because it +measurably pays. It can be switched off with `DisableQuarantine` without +touching anything else, and the router then handles an unattributable failure +entirely within the payment as it did before. + +One promotion case is knowingly left on the floor. When a probe derived lower +bound lands at exactly the amount an ambiguous failure names, a promoted bound +is written and then dropped again by the ordinary rule that a lower and an upper +bound at the same amount cannot both stand. The suspicion is still held and +still priced until then. Changing that rule would reach outside the quarantine +into bound maintenance, so it stays as it is. + +**A resumed payment's HTLCs are not counted as holds.** After a restart the +router knows a payment has attempts in flight, because the payments database +says so, but it did not choose their routes and so cannot say which interior +channels they sit on. Those shards are priced as though nothing were held, +which is where the router was before this accounting existed. + +**Searching costs more than Dijkstra does.** Every rung of the shard ladder +runs its own search, and each search may keep up to two dozen labels per node. +The graph reads are shared across the ladder and the search is bounded on hops, +labels, and total expansions, but the worst case is still well above one +shortest path query. This is the main reason the router is off by default. + +**The constants were selected, not derived.** The probability model, the retry +ladder, and the scoring weights come from an evolutionary search against a +payment simulator, scored on a real 12,000 node mainnet graph snapshot and on +synthetic topologies. They are documented for what they do rather than for why +those particular numbers are right, because for most of them nobody can say. +Some are surely fitted to the simulator that produced them. + +## Where the design came from + +The design was found by search rather than invented. We built a payment +simulator with hidden per-channel balances and real forwarding checks, then +ran an evolutionary search over whole routing algorithms, scored purely on +payment outcomes, with lnd's production stack as the baseline. Across dozens +of independent runs, every winning candidate converged on the same three +decisions: drop the per-pair penalties, drop time decay, keep per-direction +liquidity intervals. The code in this branch is a hand-written distillation +of that consensus into lnd's real payment lifecycle, hardened by several +rounds of adversarial benchmarking that each found and fixed a real bug +before the branch was called done. + +That history cuts both ways, and the limitations above say where. The +mechanisms transferred to every world the simulator could build, including a +channel graph generated outside this work entirely; the constants are the +part that may be shaped by the simulator that selected them. + +## Where the code lives + +| File | What is in it | +|---|---| +| `routing/interval_belief.go` | the interval, the quarantine, the model | +| `routing/interval_store.go` | the node wide store and its flushing | +| `routing/interval_store_sql.go` | the durable backing | +| `routing/interval_pathfind.go` | the label setting search | +| `routing/interval_session.go` | the shard ladder and the session state | +| `routing/interval_session_source.go` | session construction and the fallback | +| `routing/interval_config.go` | the search bounds and their defaults | diff --git a/docs/release-notes/release-notes-0.22.0.md b/docs/release-notes/release-notes-0.22.0.md index 750a20069cd..054f82e1a44 100644 --- a/docs/release-notes/release-notes-0.22.0.md +++ b/docs/release-notes/release-notes-0.22.0.md @@ -59,6 +59,21 @@ ## Functional Enhancements +* A new [experimental interval + router](https://github.com/lightningnetwork/lnd/pull/11048) can be selected with + `routerrpc.router=interval`. It replaces mission control with a liquidity + interval per directed channel, bounded below by the largest amount it has + watched pass and above by the smallest it has watched fail, with no time decay + anywhere. It also plans the size of an MPP shard together with the route that + carries it rather than halving its way down after a failure, and it retries a + channel at a lower amount rather than stepping around it. The default remains + unchanged, and payments to blinded paths are served by the default router + regardless of this setting. On a node running the native SQL backend the + intervals are persisted across restarts; a bound restored from disk is + applied as soft evidence with a probability floor, so that a belief which + has gone stale can still be corrected by an attempt. The algorithm is + explained in [`docs/interval_routing.md`](../interval_routing.md). + ## RPC Additions * The `routerrpc.EstimateRouteFee` RPC now supports [restricting fee estimates @@ -138,6 +153,11 @@ ## Database +* A new `liquidity_intervals` table + [stores](https://github.com/lightningnetwork/lnd/pull/11048) the liquidity + beliefs of the experimental interval router, so that they survive a restart + on nodes running the native SQL backend. + ## Code Health ## Tooling and Documentation