Repository navigation
Expand file tree
/
Copy pathoutbox.go
More file actions
1298 lines (1118 loc) · 48.3 KB
/
Copy pathoutbox.go
File metadata and controls
1298 lines (1118 loc) · 48.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
package pgoutbox
import (
"context"
"errors"
"fmt"
"math"
"math/rand/v2"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
"github.com/hatchet-dev/pgoutbox/internal/dbwrap"
"github.com/hatchet-dev/pgoutbox/sqlc"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/rs/zerolog"
)
// FlushContext is the context passed to Flusher.Flush. It embeds
// context.Context, so flushers that don't need the transaction can treat it as
// a plain context. Tx returns the transaction ProcessMessages uses to lock and
// delete messages. Flushers that write to Postgres themselves can use it so
// their writes and the outbox delete commit or roll back together.
type FlushContext interface {
context.Context
Tx() pgx.Tx
}
type flushContext struct {
context.Context
tx pgx.Tx
}
func (f *flushContext) Tx() pgx.Tx { return f.tx }
// Flusher delivers a batch of messages to their destination. Register one per
// topic with Outbox.AddFlusher. ProcessMessages calls Flush with the messages it
// has locked for the topic and deletes them only if Flush returns nil. If Flush
// returns an error, the messages stay in the outbox and are retried on the next
// call.
//
// Flushers that write to Postgres can use ctx.Tx() so that their writes and the
// outbox delete commit or roll back together.
type Flusher interface {
Flush(ctx FlushContext, msgs []*sqlc.Message) error
}
// MessageOpts describes a single message to add via Outbox.AddMessages.
type MessageOpts struct {
// Payload is the opaque message body. It is stored as-is and handed back to
// the Flusher unchanged.
Payload []byte
}
// Outbox is a transactional outbox: messages are added to a Postgres table
// within the caller's transaction and later flushed to a destination by the
// Flusher registered for their topic. Create one with NewOutbox.
//
// Topics are plain strings and don't need to be declared up front. Any number
// of topics can be used, and every topic that receives a message is tracked
// automatically.
type Outbox interface {
// AddFlusher registers the Flusher that ProcessMessages and Subscribe use to
// drain topic. Registering a flusher for a topic that already has one
// replaces it.
AddFlusher(topic string, flusher Flusher)
// AddMessages adds msgs to the topic within the caller's transaction. The
// messages become visible to ProcessMessages once tx commits, and are
// discarded if it rolls back.
//
// When the outbox was built with WithPubSub, AddMessages also notifies
// subscribers of the topic. A TxPublisher such as NewPGPubSub publishes the
// notification inside tx, so it is delivered exactly when the transaction
// commits. Other PubSub implementations need to defer the notification
// until after commit: pass a Notifier via WithNotifier and call Notify once
// the transaction has committed. Skipping the notification never loses
// messages; subscribers pick them up on their next poll.
AddMessages(ctx context.Context, tx pgx.Tx, topic string, msgs []MessageOpts, opts ...AddOpt) error
// ProcessMessages grabs a batch of messages for the topic, flushes them
// using the registered Flusher, and deletes them from the outbox in the
// same transaction if the flush succeeds. It returns the flushed messages,
// or nil if the topic was empty. The batch size defaults to 1000 and can be
// changed with WithBatchSize.
//
// Messages are locked with FOR UPDATE SKIP LOCKED, so any number of callers
// can drain a topic concurrently. If the topic has an active exclusive
// consumer, the caller must hold the lease via AcquireTopic; otherwise
// ErrExclusiveLeaseHeld or ErrExclusiveLeaseRequired is returned.
ProcessMessages(ctx context.Context, topic string, opts ...ProcessOpt) ([]*sqlc.Message, error)
// Subscribe drains the topic automatically by listening for a poll or a
// new-message notification. It runs ProcessMessages until the topic is
// empty, then waits for the poll interval (see WithPollInterval) or, when
// the outbox was built with WithPubSub, for a new-message notification, and
// drains again. It blocks until ctx is cancelled and then returns ctx.Err().
//
// If processing fails, the error is logged to the WithLogger logger and the
// messages are retried on the next wake-up. Subscribe returns an error
// immediately only if no flusher is registered for the topic, the PubSub
// subscription can't be established, or the initial lease acquisition for
// WithExclusive fails.
//
// As with ProcessMessages, a topic with an active exclusive consumer
// requires the lease. Either call AcquireTopic before Subscribe or pass
// WithExclusive to have Subscribe manage the lease itself.
Subscribe(ctx context.Context, topic string, opts ...SubscribeOpt) error
// AcquireTopic blocks until this instance holds the exclusive lease for the
// topic, then returns. A background goroutine automatically renews the
// lease until ctx is cancelled or ReleaseTopic is called, at which point
// the lease expires and another instance can take over.
//
// Once a topic has an exclusive consumer, only the lease holder may call
// ProcessMessages for it. Other instances receive ErrExclusiveLeaseHeld, and
// an instance whose lease has expired receives ErrExclusiveLeaseRequired
// until it calls AcquireTopic again.
AcquireTopic(ctx context.Context, topic string) error
// ReleaseTopic stops renewing the exclusive lease this instance holds for
// topic and expires it immediately, so a waiting instance can take over
// right away instead of waiting out the lease duration. It is a no-op if
// this instance does not hold the lease. A later ProcessMessages call for
// the topic requires AcquireTopic again.
ReleaseTopic(ctx context.Context, topic string) error
}
// ErrExclusiveLeaseHeld is returned by ProcessMessages when another outbox
// instance currently holds the exclusive lease for the topic.
var ErrExclusiveLeaseHeld = errors.New("exclusive lease held by another instance")
// ErrExclusiveLeaseRequired is returned by ProcessMessages when the topic has
// an exclusive consumer but this instance does not hold a live lease, either
// because AcquireTopic was never called or because the lease has expired.
var ErrExclusiveLeaseRequired = errors.New("exclusive lease required: call AcquireTopic first")
// defaultBatchSize is the number of messages ProcessMessages will pull per
// call when the caller has not specified WithBatchSize.
const defaultBatchSize = 1000
// AddOpt is a per-call option for AddMessages.
type AddOpt func(*addOpts)
type addOpts struct {
notifier *Notifier
}
// WithNotifier has AddMessages record its new-message notification in n so the
// caller can publish it after the transaction commits. It is only needed for
// PubSub implementations that don't implement TxPublisher, since those can't
// defer publishing to commit time. One Notifier can be shared by every
// AddMessages call in a transaction and fired once with Notify after commit.
func WithNotifier(n *Notifier) AddOpt {
return func(opts *addOpts) {
opts.notifier = n
}
}
// Notifier accumulates the new-message notifications from the AddMessages
// calls it is passed to via WithNotifier, so they can be published once the
// transaction has committed. The zero value is ready to use. Like the pgx.Tx it
// accompanies, it is not safe for concurrent use.
type Notifier struct {
hooks []func(context.Context)
}
// Notify publishes the accumulated notifications. Call it once after the
// transaction commits successfully. After a rollback, discard the Notifier
// instead. Notify is a no-op when there is nothing to publish, including when
// the PubSub implements TxPublisher and already published on the transaction.
//
// Delivery is best-effort by design: failures are logged to the WithLogger
// logger rather than returned, since subscribers fall back to polling and pick
// the messages up regardless. Calling Notify more than once only repeats the
// wake-ups, which is harmless.
func (n *Notifier) Notify(ctx context.Context) {
for _, hook := range n.hooks {
hook(ctx)
}
}
// ProcessOpt is a per-call option for ProcessMessages.
type ProcessOpt func(*processOpts)
type processOpts struct {
batchSize int32
}
func defaultProcessOpts() *processOpts {
return &processOpts{batchSize: defaultBatchSize}
}
// WithBatchSize sets the maximum number of messages ProcessMessages locks and
// hands to the Flusher in a single call. Defaults to 1000. Values that are not
// positive or that exceed math.MaxInt32 are ignored.
func WithBatchSize(n int) ProcessOpt {
return func(opts *processOpts) {
if n <= 0 || n > math.MaxInt32 {
return
}
opts.batchSize = int32(n)
}
}
// maintenanceLeaseTimeout is the age at which a held lease is considered
// abandoned and eligible for takeover by another instance. Exposed as a var
// so tests can shorten it without a config option.
var maintenanceLeaseTimeout = newAtomicDuration(30 * time.Second)
// topicScanInterval controls how often the scanner goroutine polls the topics
// table for newly registered topics. Exposed as a var for tests.
var topicScanInterval = newAtomicDuration(30 * time.Second)
// maintenanceIdleInterval is how long the maintenance goroutine sleeps when
// there are no messages at all for the topic.
const maintenanceIdleInterval = 30 * time.Second
// maintenanceRetryInterval is how long the maintenance goroutine sleeps after
// an error or after losing a lease competition.
const maintenanceRetryInterval = 5 * time.Second
// maintenanceMinInterval floors every sleep between maintenance-loop
// iterations, so a topic whose oldest message keeps hovering near the TTL
// boundary cannot re-peek in a tight loop. Exposed as a var for tests.
var maintenanceMinInterval = newAtomicDuration(5 * time.Second)
// maintenanceActivitySlack widens the window during which a topic counts as
// active beyond its TTL. It must cover the insert trigger's bump granularity
// (30s) plus at least one scan interval, so the final cleanup pass reliably
// runs before a topic goes dormant. Exposed as a var for tests.
var maintenanceActivitySlack = newAtomicDuration(2 * time.Minute)
// maintenanceCatchupInterval is how often the scanner force-includes every
// topic that still holds messages regardless of activity, recovering topics
// whose final cleanup pass was missed or whose TTL was configured after
// their last insert. Also runs on the first scan after startup. Exposed as a
// var for tests.
var maintenanceCatchupInterval = newAtomicDuration(time.Hour)
// consumerSessionRetention is how long an expired consumer-session row is
// kept before the catch-up sweep garbage-collects it. Instance ids are
// per-NewOutbox, so every process restart strands one row; sweeping them
// keeps the table at O(live instances). Deletion is semantically the same as
// expiry, so the buffer is caution against clock skew, not correctness.
// Exposed as a var for tests.
var consumerSessionRetention = newAtomicDuration(time.Hour)
// exclusiveLeaseDuration is how far ahead each consumer-session heartbeat
// pushes the session's expiry, and the length of the grace period granted
// when a holder's AcquireTopic ctx is cancelled. An instance that stops
// heartbeating loses its exclusive leases once this elapses.
var exclusiveLeaseDuration = newAtomicDuration(30 * time.Second)
// exclusiveLeaseRenewInterval controls how often the consumer-session
// heartbeat goroutine started by NewOutbox renews this instance's session.
var exclusiveLeaseRenewInterval = newAtomicDuration(10 * time.Second)
// exclusiveLeaseRetryInterval is how long AcquireTopic sleeps between attempts
// to acquire the lease when another instance currently holds it.
var exclusiveLeaseRetryInterval = newAtomicDuration(1 * time.Second)
// exclusiveLeaseWriteTimeout bounds the detached, best-effort lease writes:
// the immediate release when an exclusive Subscribe exits and the
// grace-period expiry written after a hold ends. These writes are detached
// from their caller's (usually already-cancelled) ctx, so without a bound a
// stalled database would block shutdown indefinitely; a write that cannot
// land within this window is better abandoned — the holder's consumer
// session lapsing covers the failure case.
const exclusiveLeaseWriteTimeout = 10 * time.Second
type outboxImplOpts struct {
schema string
autoMigrate bool
expirations map[string]time.Duration
defaultExpiration time.Duration
logger zerolog.Logger
pubsub PubSub
}
func defaultOpts() *outboxImplOpts {
return &outboxImplOpts{
schema: "outbox",
autoMigrate: true,
expirations: make(map[string]time.Duration),
logger: zerolog.Nop(),
}
}
type outboxImpl struct {
queries *sqlc.Queries
pool *pgxpool.Pool
schema string
instanceID uuid.UUID
// expirations holds the per-topic TTLs configured at construction time via
// WithTopicExpiration. Written to the topics table on Start so that any
// instance can discover the TTL without its own config. Immutable after NewOutbox.
expirations map[string]time.Duration
// defaultExpiration is used for topics that have no specific TTL configured
// via WithTopicExpiration. Zero means no default (those topics are skipped).
defaultExpiration time.Duration
// logger receives error-level messages from the background maintenance goroutines.
logger zerolog.Logger
// pubsub carries new-message notifications between AddMessages and
// Subscribe. Nil unless configured via WithPubSub; everything works
// without it, Subscribe just degrades to pure polling.
pubsub PubSub
// txPub is pubsub's TxPublisher facet, resolved once at NewOutbox so
// AddMessages doesn't decide notification semantics per call. Non-nil iff
// pubsub implements TxPublisher, in which case notifications are published
// on the caller's transaction instead of via the returned NotifyFunc.
txPub TxPublisher
flushers sync.Map
// exclusiveLeaseWatcher holds one goroutine per held topic that waits for
// the AcquireTopic ctx to end and then writes a single grace-period
// expiry, ending that topic's deferral to this instance's consumer
// session. AcquireTopic starts it per topic; ReleaseTopic stops it before
// clearing the lease.
exclusiveLeaseWatcher *leaseWatcher
// managed tracks the maintenance-loop goroutine for each topic whose
// maintenance lease this instance currently holds. Written only by the
// scanner goroutine; the mutex exists for test introspection.
managedMu sync.Mutex
managed map[string]*managedTopic
}
// OutboxOpt configures NewOutbox and Migrate.
type OutboxOpt func(*outboxImplOpts)
// WithSchema sets the Postgres schema that holds the outbox tables. Defaults to
// "outbox". The schema is created if it doesn't exist when migrations run.
func WithSchema(schema string) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.schema = schema
}
}
// WithAutoMigrate controls whether NewOutbox runs the embedded migrations.
// Defaults to true. Set it to false to run migrations yourself by calling
// Migrate, for example as part of a separate release step.
func WithAutoMigrate(enabled bool) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.autoMigrate = enabled
}
}
// WithTopicExpiration sets how long messages on topic are kept before the
// background maintenance goroutines delete them. NewOutbox writes the
// expiration to the topics table so that every outbox instance can discover
// it. Per-topic expirations take precedence over WithDefaultExpiration.
func WithTopicExpiration(topic string, ttl time.Duration) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.expirations[topic] = ttl
}
}
// WithDefaultExpiration sets the expiration for topics that have none
// configured via WithTopicExpiration. Topics don't need to be declared at
// startup: every topic that receives a message is tracked in the topics table
// and picks up the default automatically. Without a default, topics that have
// no explicit expiration are never expired.
func WithDefaultExpiration(ttl time.Duration) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.defaultExpiration = ttl
}
}
// WithLogger attaches a zerolog logger that receives errors from the
// background maintenance goroutines, from Subscribe's processing passes, and
// from Notifier.Notify. If not set, those errors are silent.
func WithLogger(l zerolog.Logger) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.logger = l
}
}
// WithPubSub attaches a PubSub that wakes Subscribe callers the moment new
// messages commit, instead of waiting out the poll interval. NewPGPubSub
// provides an implementation built on Postgres LISTEN/NOTIFY.
//
// Delivery is best-effort by design; if a notification is lost, Subscribe falls
// back to polling. If ps implements TxPublisher, the notification is published
// inside the AddMessages transaction. Otherwise, pass a Notifier to AddMessages
// via WithNotifier and call Notify after commit.
func WithPubSub(ps PubSub) OutboxOpt {
return func(opts *outboxImplOpts) {
opts.pubsub = ps
}
}
// NewOutbox creates an outbox backed by pool. By default it runs the embedded
// migrations, creating the outbox tables in the "outbox" schema (see WithSchema
// and WithAutoMigrate), and starts the background maintenance goroutines that
// delete expired messages. The goroutines run until ctx is cancelled, so pass a
// context tied to your application lifetime.
func NewOutbox(ctx context.Context, pool *pgxpool.Pool, fs ...OutboxOpt) (Outbox, error) {
opts := defaultOpts()
for _, f := range fs {
f(opts)
}
if err := validateSchemaName(opts.schema); err != nil {
return nil, err
}
queries := sqlc.New()
if opts.autoMigrate {
if err := runMigrations(context.Background(), pool, opts.schema); err != nil {
return nil, fmt.Errorf("could not run migrations: %w", err)
}
}
expirations := make(map[string]time.Duration, len(opts.expirations))
for k, v := range opts.expirations {
expirations[k] = v
}
o := &outboxImpl{
queries: queries,
pool: pool,
schema: opts.schema,
instanceID: uuid.New(),
expirations: expirations,
defaultExpiration: opts.defaultExpiration,
logger: opts.logger,
pubsub: opts.pubsub,
txPub: asTxPublisher(opts.pubsub),
managed: make(map[string]*managedTopic),
}
o.exclusiveLeaseWatcher = newLeaseWatcher(o.expireExclusiveLeaseAfterGrace)
for topic, ttl := range o.expirations {
if err := o.ensureTopicRow(ctx, topic, ttl); err != nil {
o.logger.Error().Err(err).Str("topic", topic).Msg("maintenance: failed to publish topic expiration")
}
}
// Lenient like ensureTopicRow: with WithAutoMigrate(false) the tables may
// not exist yet. The renewer below retries every renewal interval, and
// AcquireTopic refreshes the session transactionally itself, so a failure
// here only delays maintenance fair-share accounting.
if err := o.upsertConsumerSession(ctx); err != nil {
o.logger.Error().Err(err).Msg("exclusive consumer: failed to register consumer session")
}
go o.runConsumerSessionRenewer(ctx)
go o.runTopicScanner(ctx)
return o, nil
}
func (o *outboxImpl) AddFlusher(topic string, flusher Flusher) {
o.flushers.Store(topic, flusher)
}
func (o *outboxImpl) getFlusher(topic string) (Flusher, bool) {
f, ok := o.flushers.Load(topic)
if !ok {
return nil, false
}
flusher, ok := f.(Flusher)
if !ok {
return nil, false
}
return flusher, true
}
func (o *outboxImpl) AddMessages(ctx context.Context, tx pgx.Tx, topic string, msgs []MessageOpts, aopts ...AddOpt) error {
if topic == "" {
return fmt.Errorf("topic must not be empty")
}
if len(msgs) == 0 {
return nil
}
opts := &addOpts{}
for _, f := range aopts {
f(opts)
}
params := make([]sqlc.InsertMessageParams, len(msgs))
for i, msg := range msgs {
if len(msg.Payload) == 0 {
return fmt.Errorf("payload for topic %q at index %d must not be empty", topic, i)
}
params[i] = sqlc.InsertMessageParams{
Topic: topic,
Payload: msg.Payload,
}
}
_, err := o.queries.InsertMessage(ctx, dbwrap.New(tx, o.schema), params)
if err != nil {
return fmt.Errorf("could not insert messages for topic %q: %w", topic, err)
}
if o.txPub != nil {
// This shares the caller's transaction: a failure has aborted it, so
// surface the error rather than pretending the insert succeeded. The
// notification lands exactly when the transaction commits, leaving
// nothing for a WithNotifier Notifier to carry.
if err := o.txPub.PubInTx(ctx, tx, topic, nil); err != nil {
return fmt.Errorf("could not publish new-message notification for topic %q: %w", topic, err)
}
return nil
}
if o.pubsub != nil && opts.notifier != nil {
opts.notifier.hooks = append(opts.notifier.hooks, o.newMessageNotifier(topic))
}
return nil
}
// newMessageNotifier builds the post-commit notification hook for a generic
// (non-Tx) PubSub. Publishing out-of-band is best-effort: the messages are
// durably staged and pollers pick them up within a poll interval if the
// notification is lost, so failures are logged rather than returned.
func (o *outboxImpl) newMessageNotifier(topic string) func(context.Context) {
return func(ctx context.Context) {
if err := o.pubsub.Pub(ctx, topic, nil); err != nil {
o.logger.Error().Err(err).Str("topic", topic).Msg("subscribe: failed to publish new-message notification")
}
}
}
// asTxPublisher pins down a PubSub's notification semantics once, at
// construction: nil (making AddMessages defer to the returned NotifyFunc)
// unless ps publishes transactionally.
func asTxPublisher(ps PubSub) TxPublisher {
txp, _ := ps.(TxPublisher)
return txp
}
func (o *outboxImpl) AcquireTopic(ctx context.Context, topic string) error {
if topic == "" {
return fmt.Errorf("topic must not be empty")
}
// A watcher left over from an earlier AcquireTopic for this topic must be
// fully stopped before re-acquiring: its one action is writing an
// expiring override, which would poison the fresh session-deferred lease
// if it landed after the acquire below. Stop suppresses that write when
// it wins the race, and when it loses (the old ctx ended first) it blocks
// until the in-flight write has finished — either way the acquire always
// has the last word. Stopping the watcher makes its expiry duty ours: if
// we fail before installing a replacement, the old lease would stay
// session-deferred forever with nothing left to expire it (the heartbeat
// keeps the session fresh for the life of the process), so every failure
// return below must write the expiry the watcher no longer will.
stopped := o.exclusiveLeaseWatcher.Stop(topic)
for {
acquired, err := o.tryAcquireTopicLease(ctx, topic)
if err != nil {
o.expireLeaseIfWatcherOrphaned(ctx, topic, stopped)
return err
}
if acquired {
o.exclusiveLeaseWatcher.Start(ctx, topic)
return nil
}
select {
case <-ctx.Done():
o.expireLeaseIfWatcherOrphaned(ctx, topic, stopped)
return ctx.Err()
case <-time.After(exclusiveLeaseRetryInterval.Load()):
}
}
}
// expireLeaseIfWatcherOrphaned writes the grace-period expiry for a lease
// whose watcher was stopped but never replaced — the failure paths of
// AcquireTopic and ReleaseTopic. It restores the pre-session behavior of "the
// last timestamp lapses naturally" for a hold the caller just walked away
// from. Detached ctx because the caller's ctx being cancelled is the most
// common way to land here; if even the detached write fails, the instance's
// heartbeat is failing against the same database, so the session itself
// lapses and takes the lease with it.
func (o *outboxImpl) expireLeaseIfWatcherOrphaned(ctx context.Context, topic string, hadWatcher bool) {
if !hadWatcher {
return
}
o.expireExclusiveLeaseAfterGrace(context.WithoutCancel(ctx), topic)
}
// ReleaseTopic stops this instance's lease renewer for topic and expires the
// lease immediately in the database, so another instance can acquire it right
// away instead of waiting out the lease duration. Like a naturally expired
// lease, the topic still requires an explicit AcquireTopic call afterward -
// released is not the same as never having been exclusive.
func (o *outboxImpl) ReleaseTopic(ctx context.Context, topic string) error {
if topic == "" {
return fmt.Errorf("topic must not be empty")
}
stopped := o.exclusiveLeaseWatcher.Stop(topic)
expiredAt := time.Now().UTC().Add(-time.Second)
err := o.queries.RenewTopicExclusiveConsumer(ctx, dbwrap.New(o.pool, o.schema), sqlc.RenewTopicExclusiveConsumerParams{
Topic: topic,
ExclusiveConsumerID: &o.instanceID,
ExclusiveConsumerExpiresAt: pgtype.Timestamptz{Time: expiredAt, Valid: true},
})
if err != nil {
// The stopped watcher will never fire and the release write didn't
// land (typically: caller ctx already cancelled) — without this the
// lease would ride the session heartbeat forever. Degrades a failed
// immediate release into a natural grace-period lapse.
o.expireLeaseIfWatcherOrphaned(ctx, topic, stopped)
return fmt.Errorf("release exclusive lease for topic %q: %w", topic, err)
}
return nil
}
// tryAcquireTopicLease makes one attempt to claim the exclusive lease. Returns
// (true, nil) on success, (false, nil) if another instance currently holds a
// valid lease, or (false, err) on a database error.
func (o *outboxImpl) tryAcquireTopicLease(ctx context.Context, topic string) (bool, error) {
tx, err := o.pool.Begin(ctx)
if err != nil {
return false, fmt.Errorf("begin transaction for AcquireTopic %q: %w", topic, err)
}
defer tx.Rollback(ctx)
wrapped := dbwrap.New(tx, o.schema)
// Ensure a row exists; the trigger creates it on first message insert, but
// AcquireTopic may be called before any messages exist for this topic.
if err := o.queries.InsertTopicIfAbsent(ctx, wrapped, sqlc.InsertTopicIfAbsentParams{
Topic: topic,
ExpirationNanos: pgtype.Int8{Valid: false},
}); err != nil {
return false, fmt.Errorf("ensure topic row for %q: %w", topic, err)
}
row, err := o.queries.GetTopicForUpdate(ctx, wrapped, topic)
if err != nil {
return false, fmt.Errorf("lock topic row for %q: %w", topic, err)
}
holderIsUs := row.ExclusiveConsumerID != nil && *row.ExclusiveConsumerID == o.instanceID
leaseValid := row.ExclusiveConsumerExpiresAt.Valid && row.ExclusiveConsumerExpiresAt.Time.After(time.Now())
if row.ExclusiveConsumerID != nil && !holderIsUs && leaseValid {
return false, nil
}
// Refresh our consumer session in the same transaction: the lease written
// below is only as alive as that session, and the background heartbeat may
// not have landed yet (NewOutbox's initial upsert is best-effort — tables
// missing under WithAutoMigrate(false), transient DB error). Failing the
// acquire here beats returning success for a lease that immediately reads
// as expired.
if err := o.upsertConsumerSessionIn(ctx, wrapped); err != nil {
return false, fmt.Errorf("refresh consumer session while acquiring topic %q: %w", topic, err)
}
// The per-topic timestamp is left NULL, so the lease defers to this
// instance's consumer session for liveness (and any stale override from a
// previous holder is cleared).
if err := o.queries.SetTopicExclusiveConsumer(ctx, wrapped, sqlc.SetTopicExclusiveConsumerParams{
Topic: topic,
ExclusiveConsumerID: &o.instanceID,
ExclusiveConsumerExpiresAt: pgtype.Timestamptz{Valid: false},
}); err != nil {
return false, fmt.Errorf("set exclusive consumer for topic %q: %w", topic, err)
}
if err := tx.Commit(ctx); err != nil {
return false, fmt.Errorf("commit exclusive lease for topic %q: %w", topic, err)
}
return true, nil
}
// expireExclusiveLeaseAfterGrace writes the one-shot per-topic override that
// ends a hold: the lease stops deferring to the consumer session and instead
// lapses one lease duration from now. Called by o.exclusiveLeaseWatcher with
// an already-detached ctx once the AcquireTopic ctx has ended. The WHERE
// clause makes this a silent no-op if another instance has since taken over.
// Self-bounded so that no caller — the watcher goroutine or the
// AcquireTopic/ReleaseTopic failure paths — can hang on a stalled database.
func (o *outboxImpl) expireExclusiveLeaseAfterGrace(ctx context.Context, topic string) {
ctx, cancel := context.WithTimeout(ctx, exclusiveLeaseWriteTimeout)
defer cancel()
expiresAt := time.Now().UTC().Add(exclusiveLeaseDuration.Load())
err := o.queries.RenewTopicExclusiveConsumer(ctx, dbwrap.New(o.pool, o.schema), sqlc.RenewTopicExclusiveConsumerParams{
Topic: topic,
ExclusiveConsumerID: &o.instanceID,
ExclusiveConsumerExpiresAt: pgtype.Timestamptz{Time: expiresAt, Valid: true},
})
if err != nil {
o.logger.Error().Err(err).Str("topic", topic).Msg("exclusive consumer: failed to write lease grace-period expiry")
}
}
// upsertConsumerSession registers or refreshes this instance's row in
// consumer_sessions, the single per-instance heartbeat that exclusive and
// maintenance leases defer to for liveness.
func (o *outboxImpl) upsertConsumerSession(ctx context.Context) error {
return o.upsertConsumerSessionIn(ctx, dbwrap.New(o.pool, o.schema))
}
func (o *outboxImpl) upsertConsumerSessionIn(ctx context.Context, wrapped *dbwrap.Wrapper) error {
expiresAt := time.Now().UTC().Add(exclusiveLeaseDuration.Load())
return o.queries.UpsertConsumerSession(ctx, wrapped, sqlc.UpsertConsumerSessionParams{
ConsumerID: o.instanceID,
ExpiresAt: pgtype.Timestamptz{Time: expiresAt, Valid: true},
MaintainsDefaultExpiration: o.defaultExpiration > 0,
})
}
// runConsumerSessionRenewer keeps this instance's consumer session fresh for
// the life of the outbox — one UPDATE per renewal interval regardless of how
// many topics are held.
func (o *outboxImpl) runConsumerSessionRenewer(ctx context.Context) {
for {
select {
case <-ctx.Done():
return
case <-time.After(exclusiveLeaseRenewInterval.Load()):
}
if err := o.upsertConsumerSession(ctx); err != nil && !errors.Is(err, context.Canceled) {
o.logger.Error().Err(err).Msg("exclusive consumer: failed to renew consumer session")
}
}
}
// checkExclusiveAccessForUpdate locks the topic row (FOR UPDATE OF t — the
// joined consumer_sessions row is deliberately left unlocked, since the
// holder's every other topic reads it too) within the caller's transaction
// and returns an error if another instance holds an active exclusive lease.
// A missing topics row (topic not yet seen) is treated as non-exclusive.
//
// The row lock is held until the caller's transaction commits or rolls back,
// so a concurrent AcquireTopic or lease renewal for this topic blocks until
// then instead of being able to hand the lease to another instance while this
// transaction is still relying on the check having passed. That closes the
// gap between validating ownership and committing the work done under it.
func (o *outboxImpl) checkExclusiveAccessForUpdate(ctx context.Context, wrapped *dbwrap.Wrapper, topic string) error {
row, err := o.queries.GetTopicForUpdate(ctx, wrapped, topic)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil
}
return fmt.Errorf("check exclusive access for topic %q: %w", topic, err)
}
if row.ExclusiveConsumerID == nil {
return nil
}
if *row.ExclusiveConsumerID == o.instanceID {
if row.ExclusiveConsumerExpiresAt.Valid && row.ExclusiveConsumerExpiresAt.Time.After(time.Now()) {
return nil
}
return fmt.Errorf("exclusive lease for topic %q has expired: %w", topic, ErrExclusiveLeaseRequired)
}
if row.ExclusiveConsumerExpiresAt.Valid && row.ExclusiveConsumerExpiresAt.Time.After(time.Now()) {
return ErrExclusiveLeaseHeld
}
// Another instance held the lease but it has expired; require explicit acquire.
return fmt.Errorf("exclusive access required for topic %q: %w", topic, ErrExclusiveLeaseRequired)
}
func (o *outboxImpl) ProcessMessages(ctx context.Context, topic string, popts ...ProcessOpt) ([]*sqlc.Message, error) {
f, ok := o.getFlusher(topic)
if !ok {
return nil, fmt.Errorf("no flusher registered for topic %q", topic)
}
po := defaultProcessOpts()
for _, opt := range popts {
opt(po)
}
tx, err := o.pool.Begin(ctx)
if err != nil {
return nil, fmt.Errorf("could not begin transaction for processing messages for topic %q: %w", topic, err)
}
defer tx.Rollback(ctx)
wrapped := dbwrap.New(tx, o.schema)
if err := o.checkExclusiveAccessForUpdate(ctx, wrapped, topic); err != nil {
return nil, err
}
msgs, err := o.queries.AcquireMessagesByTopic(ctx, wrapped, sqlc.AcquireMessagesByTopicParams{
Topic: topic,
Limit: pgtype.Int4{
Int32: po.batchSize,
Valid: true,
},
})
if err != nil {
return nil, fmt.Errorf("could not acquire messages for topic %q: %w", topic, err)
}
if len(msgs) == 0 {
// just commit to avoid rollback monitoring
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("could not commit transaction for processing messages for topic %q: %w", topic, err)
}
return nil, nil
}
err = f.Flush(&flushContext{Context: ctx, tx: tx}, msgs)
if err != nil {
return nil, fmt.Errorf("flusher failed for topic %q: %w", topic, err)
}
// delete the messages that were flushed
msgIDs := make([]int64, len(msgs))
for i, msg := range msgs {
msgIDs[i] = msg.ID
}
err = o.queries.DeleteMessagesByIds(ctx, wrapped, sqlc.DeleteMessagesByIdsParams{
Topic: topic,
Ids: msgIDs,
})
if err != nil {
return nil, fmt.Errorf("could not delete messages for topic %q: %w", topic, err)
}
// This transaction holds the topic row lock (checkExclusiveAccessForUpdate)
// for its whole lifetime, which makes the insert trigger's SKIP LOCKED
// activity bump skip this topic — on a continuously-processed topic the
// trigger could be starved out of every bump and the topic would read as
// dormant despite live traffic. Since we just processed messages and
// already hold the lock that causes the starvation, refresh the activity
// stamp here; the query self-gates to the trigger's 30s granularity, so
// this is at most one extra row write per topic per window.
if err := o.queries.BumpTopicActivityIfStale(ctx, wrapped, topic); err != nil {
return nil, fmt.Errorf("could not bump activity for topic %q: %w", topic, err)
}
if err := tx.Commit(ctx); err != nil {
return nil, fmt.Errorf("could not commit transaction for processing messages for topic %q: %w", topic, err)
}
return msgs, nil
}
// managedTopic tracks one running maintenance loop. dormant distinguishes a
// scanner-initiated stop (topic left the active window: final cleanup pass
// and lease release on the way out) from plain shutdown (exit with zero
// writes; the consumer session lapsing is the release).
type managedTopic struct {
cancel context.CancelFunc
done chan struct{}
dormant atomic.Bool
}
// runTopicScanner periodically scans for active topics (last insert within
// TTL + slack), claims a fair share of their maintenance leases for this
// instance, and manages one maintenance goroutine per claimed topic. Claimed
// leases carry no expiry of their own: they stay valid exactly as long as
// this instance's consumer session, so holding them costs zero per-topic
// writes and a crashed instance's topics free up as soon as its session
// lapses. Dormant topics cost nothing on any instance: no goroutine, no
// peek queries, no lease row.
func (o *outboxImpl) runTopicScanner(ctx context.Context) {
var lastCatchup time.Time
for {
if ctx.Err() != nil {
return
}
lastCatchup = o.scanTopics(ctx, lastCatchup)
select {
case <-ctx.Done():
return
case <-time.After(topicScanInterval.Load()):
}
}
}
// scanTopics runs one scanner tick and returns the updated time of the last
// successful catch-up sweep.
func (o *outboxImpl) scanTopics(ctx context.Context, lastCatchup time.Time) time.Time {
wrapped := dbwrap.New(o.pool, o.schema)
o.reapExitedLoops()
// The catch-up sweep force-includes every topic that still holds
// messages, active or not: it recovers topics whose final cleanup pass
// was missed (e.g. the whole fleet was down at the wrong moment) and
// topics whose TTL was configured after their last insert was already
// older than it. The zero time on the first tick makes it always run at
// startup.
include := []string{}
if time.Since(lastCatchup) >= maintenanceCatchupInterval.Load() {
topics, err := o.queries.GetTopicsWithMessages(ctx, wrapped)
if err != nil {
if !errors.Is(err, context.Canceled) {
o.logger.Error().Err(err).Msg("maintenance: failed to list topics with messages for catch-up sweep")
}
} else {
include = topics
lastCatchup = time.Now()
}
// Best-effort GC of session rows stranded by dead instances, on the
// same cadence. Every instance runs it; the guarded DELETE makes the
// overlap harmless.
cutoff := time.Now().UTC().Add(-consumerSessionRetention.Load())
if err := o.queries.DeleteExpiredConsumerSessions(ctx, wrapped,
pgtype.Timestamptz{Time: cutoff, Valid: true}); err != nil && !errors.Is(err, context.Canceled) {
o.logger.Error().Err(err).Msg("maintenance: failed to delete expired consumer sessions")
}
}
defaultNanos := pgtype.Int8{}
if o.defaultExpiration > 0 {
defaultNanos = pgtype.Int8{Int64: o.defaultExpiration.Nanoseconds(), Valid: true}
}
candidates, err := o.queries.GetMaintenanceCandidateTopics(ctx, wrapped, sqlc.GetMaintenanceCandidateTopicsParams{
HolderID: o.instanceID,
StaleCutoff: pgtype.Timestamptz{Time: time.Now().UTC().Add(-maintenanceLeaseTimeout.Load()), Valid: true},
DefaultExpirationNanos: defaultNanos,
IncludeTopics: include,
SlackSeconds: maintenanceActivitySlack.Load().Seconds(),
})
if err != nil {
if !errors.Is(err, context.Canceled) {
o.logger.Error().Err(err).Msg("maintenance: failed to scan topics table")
}
return lastCatchup
}
ttls := make(map[string]time.Duration, len(candidates))
var held []string
var explicit, dflt claimClass
for _, c := range candidates {
ttl := o.defaultExpiration
class := &dflt
if c.ExpirationNanos.Valid {
ttl = time.Duration(c.ExpirationNanos.Int64)
class = &explicit
}
if ttl <= 0 {
continue
}
ttls[c.Topic] = ttl
class.candidates++
switch {
case c.HeldByMe:
class.held++
held = append(held, c.Topic)
case c.Unleased:
class.unleased = append(class.unleased, c.Topic)
case c.LeaseExpired:
class.stale = append(class.stale, c.Topic)
}
}
held = append(held, o.claimTopics(ctx, wrapped, &explicit, &dflt)...)
o.reconcileLoops(ctx, ttls, held)
return lastCatchup
}
// claimClass groups one scanner tick's claim state for one fair-share class