peer: bound peer-controlled resource growth

docs/release-notes/release-notes-0.20.4.md

@@ -21,6 +21,10 @@
 
 # Bug Fixes
 
+* Peer connections [now rate limit inbound ping replies and bound outgoing
+  message queue growth](https://github.com/lightningnetwork/lnd/pull/11090),
+  preventing peer-controlled resource exhaustion.
+
 * Channel funding attempts [now return
   cleanly](https://github.com/lightningnetwork/lnd/pull/11035) when their
   pending wallet reservation is no longer present.

docs/release-notes/release-notes-0.21.3.md

@@ -21,6 +21,10 @@
 
 # Bug Fixes
 
+* Peer connections [now rate limit inbound ping replies and bound outgoing
+  message queue growth](https://github.com/lightningnetwork/lnd/pull/11090),
+  preventing peer-controlled resource exhaustion.
+
 * Channel funding attempts [now return
   cleanly](https://github.com/lightningnetwork/lnd/pull/11035) when their
   pending wallet reservation is no longer present.

peer/brontide.go

@@ -59,6 +59,7 @@ import (
 	"github.com/lightningnetwork/lnd/ticker"
 	"github.com/lightningnetwork/lnd/tlv"
 	"github.com/lightningnetwork/lnd/watchtower/wtclient"
+	"golang.org/x/time/rate"
 )
 
 const (
@@ -99,6 +100,40 @@ const (
 	// needlessly wasteful of precious Tor bandwidth for little to no gain.
 	pongSizeCeiling = 4096
 
+	// pongReplyRate is the sustained number of pings per second that we
+	// answer. Our own keepalive runs once per minute, and even aggressive
+	// implementations use intervals around ten seconds, so one reply per
+	// second leaves ample headroom for honest peers.
+	pongReplyRate = 1.0
+
+	// pongReplyBurst absorbs ping clumps after reconnects or over
+	// high-latency circuits before pong replies are rate limited.
+	pongReplyBurst = 20
+
+	// pingFloodRate is the sustained number of pings per second tolerated
+	// before we disconnect a flooding peer.
+	pingFloodRate = 10.0
+
+	// pingFloodBurst tolerates short bursts before the flood rate applies.
+	pingFloodBurst = 200
+
+	// maxQueuedMsgs is the maximum number of outgoing messages buffered for
+	// a peer before its connection is torn down.
+	maxQueuedMsgs = 10000
+
+	// maxQueuedBytes is the approximate amount of explicitly charged queue
+	// memory one peer may retain. This is the softest tuning choice here;
+	// unenumerated dynamic message data remains protected by maxQueuedMsgs,
+	// so this is an accounting cap rather than a strict heap ceiling.
+	maxQueuedBytes = 16 << 20
+
+	// queuedMsgOverhead charges every queued message for its fixed cost. A
+	// retained Pong was measured at 104 bytes across its Pong allocation,
+	// outgoingMsg interface box, and list element, which we round up here.
+	// Charging this overhead makes cheap-message floods visible to the byte
+	// budget instead of treating shared Pong payloads as free.
+	queuedMsgOverhead = 128
+
 	// torTimeoutMultiplier is the scaling factor we use on network timeouts
 	// for Tor peers.
 	torTimeoutMultiplier = 3
@@ -585,6 +620,11 @@ type Brontide struct {
 
 	pingManager *PingManager
 
+	// pongReplyLimiter and pingFloodLimiter enforce the two-tier inbound
+	// ping policy for this connection.
+	pongReplyLimiter *rate.Limiter
+	pingFloodLimiter *rate.Limiter
+
 	// lastPingPayload stores an unsafe pointer wrapped as an atomic
 	// variable which points to the last payload the remote party sent us
 	// as their ping.
@@ -746,6 +786,12 @@ func NewBrontide(cfg Config) *Brontide {
 		activeSignal:  make(chan struct{}),
 		sendQueue:     make(chan outgoingMsg),
 		outgoingQueue: make(chan outgoingMsg),
+		pongReplyLimiter: rate.NewLimiter(
+			pongReplyRate, pongReplyBurst,
+		),
+		pingFloodLimiter: rate.NewLimiter(
+			pingFloodRate, pingFloodBurst,
+		),
 		addedChannels: &lnutils.SyncMap[lnwire.ChannelID, struct{}]{},
 		activeChannels: &lnutils.SyncMap[
 			lnwire.ChannelID, *lnwallet.LightningChannel,
@@ -2344,6 +2390,22 @@ out:
 			p.pingManager.ReceivedPong(msg)
 
 		case *lnwire.Ping:
+			// A flood is a flood even when the requested Pong size
+			// would normally cause us to ignore the Ping.
+			if !p.pingFloodLimiter.Allow() {
+				err := errors.New("ping flood limit exceeded")
+				p.storeError(err)
+				p.log.Warnf("%v", err)
+
+				// queueHandler still runs.
+				// It services outgoingQueue.
+				// Disconnect lets the ping manager finish.
+				// Teardown follows.
+				p.Disconnect(err)
+
+				break out
+			}
+
 			// First, we'll store their latest ping payload within
 			// the relevant atomic variable.
 			p.lastPingPayload.Store(msg.PaddingBytes[:])
@@ -2355,6 +2417,15 @@ out:
 				continue
 			}
 
+			// BOLT 1 requires a Pong for every Ping below the size
+			// ceiling. We deliberately deviate above this rate: its
+			// rationale recommends limited precautions against Ping
+			// flooding. No honest cadence comes near this limit.
+			if !p.pongReplyLimiter.Allow() {
+				p.log.Debugf("Pong reply rate limited")
+				continue
+			}
+
 			// Next, we'll send over the amount of specified pong
 			// bytes.
 			pong := lnwire.NewPong(p.cfg.PongBuf[0:msg.NumPongBytes])
@@ -3094,6 +3165,35 @@ func (p *Brontide) queueHandler() {
 	// been queued. This predominately includes messages from the gossiper.
 	lazyMsgs := list.New()
 
+	var (
+		numQueued   int
+		queuedBytes int
+	)
+
+	// pushMsg buffers a message on the list matching its priority, and
+	// reports whether the combined queue remains within its bounds.
+	pushMsg := func(msg outgoingMsg) bool {
+		if msg.priority {
+			priorityMsgs.PushBack(msg)
+		} else {
+			lazyMsgs.PushBack(msg)
+		}
+
+		numQueued++
+		queuedBytes += msgQueueCost(msg.msg)
+
+		return numQueued <= maxQueuedMsgs &&
+			queuedBytes <= maxQueuedBytes
+	}
+
+	// popMsg accounts for a message leaving the combined queue. The cost is
+	// recomputed from the same message charged on entry, so queued messages
+	// must not be mutated while buffered.
+	popMsg := func(msg outgoingMsg) {
+		numQueued--
+		queuedBytes -= msgQueueCost(msg.msg)
+	}
+
 	for {
 		// Examine the front of the priority queue, if it is empty check
 		// the low priority queue.
@@ -3117,11 +3217,14 @@ func (p *Brontide) queueHandler() {
 				} else {
 					lazyMsgs.Remove(elem)
 				}
+				popMsg(front)
 			case msg := <-p.outgoingQueue:
-				if msg.priority {
-					priorityMsgs.PushBack(msg)
-				} else {
-					lazyMsgs.PushBack(msg)
+				if !pushMsg(msg) {
+					p.failQueueOverflow(
+						numQueued, queuedBytes,
+					)
+
+					return
 				}
 			case <-p.cg.Done():
 				return
@@ -3132,10 +3235,12 @@ func (p *Brontide) queueHandler() {
 			// into the queue from outside sub-systems.
 			select {
 			case msg := <-p.outgoingQueue:
-				if msg.priority {
-					priorityMsgs.PushBack(msg)
-				} else {
-					lazyMsgs.PushBack(msg)
+				if !pushMsg(msg) {
+					p.failQueueOverflow(
+						numQueued, queuedBytes,
+					)
+
+					return
 				}
 			case <-p.cg.Done():
 				return
@@ -3144,6 +3249,77 @@ func (p *Brontide) queueHandler() {
 	}
 }
 
+// msgQueueCost returns the approximate retained memory charged to an outgoing
+// message without serializing it. Under-counting a type weakens the byte
+// budget, but maxQueuedMsgs still caps the raw count. CommitSig.HtlcSigs is the
+// known material undercount; commitment flow control bounds it by channel
+// count rather than allowing a peer to flood it in bulk.
+func msgQueueCost(msg lnwire.Message) int {
+	switch msg := msg.(type) {
+	// A Pong aliases one server-wide buffer, so its payload costs no
+	// additional retained queue memory.
+	case *lnwire.Pong:
+		return queuedMsgOverhead
+
+	// A failure reason is preserved byte-for-byte when forwarded upstream
+	// and is the largest variable payload a remote peer can drive in bulk.
+	case *lnwire.UpdateFailHTLC:
+		return queuedMsgOverhead + len(msg.Reason)
+
+	// The onion packet is inline rather than a slice, so charge it in
+	// addition to any separately retained extra data.
+	case *lnwire.UpdateAddHTLC:
+		return queuedMsgOverhead + lnwire.OnionPacketSize +
+			len(msg.ExtraData)
+
+	// Error and Warning retain peer-controlled diagnostic payloads, so
+	// charge their backing bytes against the queue memory limit.
+	case *lnwire.Error:
+		return queuedMsgOverhead + len(msg.Data)
+
+	case *lnwire.Warning:
+		return queuedMsgOverhead + len(msg.Data)
+
+	// Unknown message types retain only their fixed interface and list
+	// storage here; the independent count cap bounds their total volume.
+	default:
+		return queuedMsgOverhead
+	}
+}
+
+// failQueueOverflow tears the connection down after the peer's outgoing
+// message queue has grown past its bounds, then keeps that queue serviced
+// until teardown completes. We disconnect rather than drop or block: dropping
+// would punch a hole in an ordered protocol stream, while blocking would push
+// backpressure onto whichever subsystem happened to be sending.
+//
+// NOTE: This blocks until the peer's context is cancelled, so it must be
+// called from the queueHandler goroutine itself.
+func (p *Brontide) failQueueOverflow(numQueued, queuedBytes int) {
+	err := fmt.Errorf("outgoing message queue exceeded bounds: "+
+		"messages=%d, bytes=%d", numQueued, queuedBytes)
+	p.storeError(err)
+	p.log.Warnf("%v", err)
+
+	// Disconnect gets its own goroutine because we have to keep draining.
+	// Every message producer parks on outgoingQueue until the peer context
+	// is cancelled, and Disconnect waits on the ping manager before that
+	// cancellation. Walking away now could wedge both goroutines.
+	go p.Disconnect(err)
+
+	for {
+		select {
+		case msg := <-p.outgoingQueue:
+			if msg.errChan != nil {
+				msg.errChan <- lnpeer.ErrPeerExiting
+			}
+
+		case <-p.cg.Done():
+			return
+		}
+	}
+}
+
 // PingTime returns the estimated ping time to the peer in microseconds.
 func (p *Brontide) PingTime() int64 {
 	return p.pingManager.GetPingTimeMicroSeconds()