Skip to content

Commit 2c86aec

Browse files
committed
fix(kafka): keep dequeue's deadline while skipping tombstones
Skipping a tombstone went straight back to poll and never reached the deadline check, so a steady stream of tombstones held one dequeue() for as long as the stream lasted (8 s for an 8 s stream). Both clients now stop at the deadline while skipping. The raw client also holds the broker's long-poll wait to the time left once it is past a tombstone, so an empty topic behind tombstones no longer costs the full 5 s, and it steps over an empty value as it does a null one. Nothing is committed on the tombstone path. Acknowledging the next record commits past the skipped tombstones, so a restart gets past them. Tests: the deadline under a tombstone stream (both clients); the raw deadline at the end of the topic and through a backlog; a record arriving during the capped wait is still delivered; the raw client steps over an empty value; the record key after a skip; a restart gets past a tombstone through the normal acknowledge. Signed-off-by: Michael <michael@codeinfinity.co.za>
1 parent 82b62d8 commit 2c86aec

2 files changed

Lines changed: 301 additions & 71 deletions

File tree

‎tests/test_queue_kafka_tombstone.py‎

Lines changed: 223 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -170,34 +170,246 @@ def test_raw_path_reads_past_a_tombstone():
170170
], one_per_batch=True)
171171
reader = _raw_connector("tina4_tomb_raw_" + secrets.token_hex(8))
172172
try:
173-
got = [reader.dequeue(topic), reader.dequeue(topic)]
174-
assert [m and m.get("n") for m in got] == [1, 2], got
173+
got = [reader.dequeue_record(topic), reader.dequeue_record(topic)]
174+
assert [m and m.get("n") for m, _ in got] == [1, 2], got
175+
# The key names the delivered record's own offset, not the tombstone's.
176+
assert [key for _, key in got] == [f"{topic}:0:0", f"{topic}:0:2"], got
177+
finally:
178+
reader.close()
179+
180+
181+
def test_raw_path_still_waits_out_its_deadline_for_a_record_after_tombstones():
182+
"""Past a tombstone the fetch still long-polls, only for the time left.
183+
184+
Capping the broker's wait must not turn it into no wait: a record produced
185+
while the capped fetch waits is delivered by that same dequeue().
186+
"""
187+
import threading
188+
import time
189+
190+
_require_broker()
191+
from confluent_kafka import Producer
192+
193+
topic = _produce([(b"k0", {"n": 0}), (b"k1", None), (b"k1", None)], one_per_batch=True)
194+
reader = _raw_connector("tina4_tomb_rawwait_" + secrets.token_hex(8))
195+
try:
196+
assert (reader.dequeue(topic) or {}).get("n") == 0
197+
# Connected before the wait starts, so only the send lands inside it.
198+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
199+
producer.list_topics(topic, timeout=15)
200+
def late():
201+
time.sleep(0.4)
202+
producer.produce(topic, value=json.dumps({"n": "late"}).encode(), partition=0)
203+
producer.flush(15)
204+
sender = threading.Thread(target=late, daemon=True)
205+
sender.start()
206+
started = time.monotonic()
207+
message, key = reader.dequeue_record(topic)
208+
took = time.monotonic() - started
209+
sender.join(20)
210+
assert message == {"n": "late"}, f"the capped fetch did not wait for the record: {message!r}"
211+
assert key == f"{topic}:0:3", key
212+
assert took < 1.6, f"dequeue() took {took:.1f}s (deadline ~1s)"
175213
finally:
176214
reader.close()
177215

178216

179217
def test_raw_path_cannot_spin_on_a_tombstone_it_cannot_step_past():
180218
"""A tombstone batched ahead of a record must not loop the socket path.
181219
182-
The fetch hands back the whole batch and the decoder reads its first record,
183-
so the tombstone can come back at an offset the reader already passed. The
184-
skip loop must return rather than fetch the same offset for ever. Run in a
185-
thread with a cap so a regression fails here instead of hanging the suite.
220+
The fetch hands back the whole batch and the raw decoder reads only its
221+
first record, so the tombstone comes back at an offset the reader already
222+
passed. The skip loop must return at once rather than fetch the same offset
223+
until the deadline. (The record after it is still not delivered by the raw
224+
path: that is the separate batch-decoding limit, not covered here.)
186225
"""
187226
import threading
227+
import time
188228

189229
_require_broker()
190230
topic = _produce([(b"k1", None), (b"k2", {"n": 1})]) # one batch
191231
reader = _raw_connector("tina4_tomb_spin_" + secrets.token_hex(8))
192-
outcome = []
193-
worker = threading.Thread(
194-
target=lambda: outcome.extend([reader.dequeue(topic), reader.dequeue(topic)]),
195-
daemon=True,
196-
)
232+
took = []
233+
def run():
234+
for _ in range(2):
235+
started = time.monotonic()
236+
reader.dequeue(topic)
237+
took.append(time.monotonic() - started)
238+
worker = threading.Thread(target=run, daemon=True)
197239
try:
198240
worker.start()
199241
worker.join(30)
200242
assert not worker.is_alive(), "dequeue() spun on the tombstone for 30s"
243+
# The first call includes the metadata round trip; the second must not
244+
# sit re-fetching the same offset for the whole ~1 s deadline.
245+
assert took[1] < 0.5, f"dequeue() re-fetched the same offset for {took[1]:.2f}s"
201246
finally:
202247
if not worker.is_alive():
203248
reader.close()
249+
250+
251+
def _committed(topic, group_id):
252+
"""The group's durably committed offset for partition 0 (negative = none)."""
253+
from confluent_kafka import Consumer, TopicPartition
254+
255+
probe = Consumer({"bootstrap.servers": KAFKA_BROKERS, "group.id": group_id,
256+
"enable.auto.commit": False})
257+
try:
258+
return probe.committed([TopicPartition(topic, 0)], timeout=15)[0].offset
259+
finally:
260+
probe.close()
261+
262+
263+
@pytest.mark.parametrize("client", ["confluent", "raw"])
264+
def test_dequeue_keeps_its_deadline_under_a_stream_of_tombstones(client):
265+
"""Skipping tombstones must not hold dequeue() past its ~1 s deadline.
266+
267+
The skip went straight back to poll and never reached the deadline check, so
268+
while tombstones kept arriving (a burst of CDC deletes) one dequeue() blocked
269+
for as long as the stream lasted: 8 s for an 8 s stream.
270+
"""
271+
import threading
272+
import time
273+
274+
_require_broker()
275+
from confluent_kafka import Producer
276+
277+
topic = _produce([(b"k0", {"n": 0})])
278+
group = "tina4_tomb_deadline_" + secrets.token_hex(8)
279+
reader = (KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
280+
if client == "confluent" else _raw_connector(group))
281+
try:
282+
assert (reader.dequeue(topic) or {}).get("n") == 0 # warm: joined + assigned
283+
stop = time.monotonic() + 6
284+
def feed():
285+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
286+
while time.monotonic() < stop:
287+
producer.produce(topic, value=None, partition=0)
288+
producer.flush(5)
289+
time.sleep(0.1)
290+
threading.Thread(target=feed, daemon=True).start()
291+
time.sleep(0.3)
292+
started = time.monotonic()
293+
outcome = []
294+
worker = threading.Thread(target=lambda: outcome.append(reader.dequeue(topic)), daemon=True)
295+
worker.start()
296+
worker.join(20)
297+
took = time.monotonic() - started
298+
assert not worker.is_alive(), "dequeue() never returned under the tombstone stream"
299+
assert outcome == [None]
300+
assert took < 1.6, f"dequeue() took {took:.1f}s under a tombstone stream (deadline ~1s)"
301+
finally:
302+
reader.close()
303+
304+
305+
def test_raw_path_keeps_its_deadline_when_the_tombstones_end_the_topic():
306+
"""Past a tombstone the broker's long-poll wait is held to the deadline too.
307+
308+
The fetch asks the broker to wait up to 5 s for data. After stepping past
309+
tombstones to the end of the topic, the next fetch sat out the whole 5 s,
310+
so dequeue() took 5 s instead of its ~1 s.
311+
"""
312+
import time
313+
314+
_require_broker()
315+
topic = _produce([(b"k", {"n": 0})])
316+
reader = _raw_connector("tina4_tomb_rawtail_" + secrets.token_hex(8))
317+
try:
318+
assert (reader.dequeue(topic) or {}).get("n") == 0
319+
from confluent_kafka import Producer
320+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
321+
for _ in range(3):
322+
producer.produce(topic, value=None, partition=0)
323+
producer.flush(15)
324+
started = time.monotonic()
325+
assert reader.dequeue(topic) is None
326+
took = time.monotonic() - started
327+
assert took < 1.6, f"dequeue() took {took:.1f}s past the tombstones (deadline ~1s)"
328+
# The offset moved past them, so the next call does not rescan them.
329+
assert reader._offsets[topic] == 4
330+
finally:
331+
reader.close()
332+
333+
334+
def test_raw_path_steps_over_an_empty_value():
335+
"""The raw decoder treats an empty value like a null one and steps past it.
336+
337+
Both used to stop the reader on the spot for ever. (The confluent path is
338+
unchanged for an empty value: it returns no message and moves on.)
339+
"""
340+
_require_broker()
341+
from confluent_kafka import Producer
342+
343+
topic = "tina4_tombstone_" + secrets.token_hex(6)
344+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
345+
producer.produce(topic, value=b"", partition=0)
346+
producer.flush(15)
347+
producer.produce(topic, value=json.dumps({"n": 1}).encode(), partition=0)
348+
producer.flush(15)
349+
reader = _raw_connector("tina4_tomb_empty_" + secrets.token_hex(8))
350+
try:
351+
assert (reader.dequeue(topic) or {}).get("n") == 1
352+
finally:
353+
reader.close()
354+
355+
356+
def test_raw_path_keeps_its_deadline_through_a_backlog_of_tombstones():
357+
"""A backlog of tombstones longer than one deadline is cut off at it.
358+
359+
Each fetch of a tombstone returns at once, so only the deadline check stops
360+
the loop: without it one dequeue() walked the whole backlog.
361+
"""
362+
import time
363+
364+
from confluent_kafka import Producer
365+
366+
_require_broker()
367+
topic = "tina4_tombstone_" + secrets.token_hex(6)
368+
# One record per batch, so the raw decoder can step over every one.
369+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS,
370+
"batch.num.messages": 1, "linger.ms": 0})
371+
total = 6000
372+
for i in range(total):
373+
producer.produce(topic, value=None, partition=0)
374+
if i % 500 == 0:
375+
producer.poll(0)
376+
producer.flush(60)
377+
reader = _raw_connector("tina4_tomb_backlog_" + secrets.token_hex(8))
378+
try:
379+
started = time.monotonic()
380+
assert reader.dequeue(topic) is None
381+
took = time.monotonic() - started
382+
assert took < 1.6, f"dequeue() took {took:.1f}s on a tombstone backlog (deadline ~1s)"
383+
assert 0 < reader._offsets[topic] < total, reader._offsets
384+
finally:
385+
reader.close()
386+
387+
388+
def test_a_fresh_process_gets_past_a_tombstone_through_the_normal_ack():
389+
"""Nothing commits on the tombstone path; acknowledging the record after it
390+
commits past both, so a restart in the same group neither stops at the
391+
tombstone nor redelivers anything."""
392+
_require_broker()
393+
topic = _produce([(b"k1", {"n": 1}), (b"k1", None), (b"k2", {"n": 2})])
394+
group = "tina4_tomb_restart_" + secrets.token_hex(8)
395+
first = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
396+
try:
397+
message, key = first.dequeue_record(topic)
398+
assert message == {"n": 1}
399+
first.acknowledge(topic, key)
400+
assert _committed(topic, group) == 1 # at the tombstone
401+
finally:
402+
first.close()
403+
second = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
404+
try:
405+
message, key = second.dequeue_record(topic)
406+
assert message == {"n": 2}, f"a restart did not get past the tombstone: {message!r}"
407+
second.acknowledge(topic, key)
408+
assert _committed(topic, group) == 3
409+
finally:
410+
second.close()
411+
third = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
412+
try:
413+
assert third.dequeue_record(topic) == (None, None)
414+
finally:
415+
third.close()

0 commit comments

Comments
 (0)