Skip to content

Commit 4b159b8

Browse files
Merge pull request #309 from tina4stack/fix/kafka-tombstone-skip-not-stop
fix(kafka): read past a tombstone instead of stopping at it
2 parents a0e846d + 2c86aec commit 4b159b8

2 files changed

Lines changed: 394 additions & 58 deletions

File tree

‎tests/test_queue_kafka_tombstone.py‎

Lines changed: 307 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,20 +43,35 @@ def _require_broker():
4343
pytest.skip("kafka not set: export TINA4_KAFKA_BROKERS (e.g. 127.0.0.1:9092)")
4444

4545

46-
def _produce(records):
46+
def _produce(records, one_per_batch=False):
4747
"""Produce (key, value) pairs with the REAL confluent Producer. A value of
48-
None is a genuine tombstone on the wire, exactly like a Debezium delete."""
48+
None is a genuine tombstone on the wire, exactly like a Debezium delete.
49+
50+
one_per_batch flushes after every record so each lands in its own record
51+
batch: the raw socket path decodes only the first record of a fetched batch.
52+
"""
4953
from confluent_kafka import Producer
5054

5155
topic = "tina4_tombstone_" + secrets.token_hex(6)
5256
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
5357
for key, value in records:
5458
payload = None if value is None else json.dumps(value).encode("utf-8")
55-
producer.produce(topic, key=key, value=payload)
59+
producer.produce(topic, key=key, value=payload, partition=0)
60+
if one_per_batch:
61+
producer.flush(15)
5662
producer.flush(15)
5763
return topic
5864

5965

66+
def _raw_connector(group_id):
67+
"""The zero-dependency socket path, selected the way TestKafkaRawPathLive
68+
does: the flag the constructor sets from its import probe."""
69+
reader = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group_id)
70+
reader._use_confluent = False
71+
reader._confluent = None
72+
return reader
73+
74+
6075
def _drain(reader, topic, want_key, max_calls=12):
6176
"""Call dequeue up to max_calls times, returning the first message whose
6277
'event' equals want_key (or None). Never swallows an exception -- a crash in
@@ -109,3 +124,292 @@ def test_only_a_tombstone_yields_no_job_and_no_crash():
109124
assert reader.dequeue(topic) is None
110125
finally:
111126
reader.close()
127+
128+
129+
def test_single_pass_drain_reads_past_a_tombstone(monkeypatch):
130+
"""consume(poll_interval=0) drains to the end of the topic, tombstone or not.
131+
132+
Answering a tombstone with "no message" reads as an empty queue, so a
133+
single-pass drain stopped at it and left every later record behind; a fresh
134+
process (a cron drain) resumed at the tombstone's committed offset and
135+
stopped there again, every run.
136+
"""
137+
_require_broker()
138+
from tina4_python.queue import Queue
139+
140+
topic = _produce([
141+
(b"k1", {"n": 1}),
142+
(b"k1", None),
143+
(b"k2", {"n": 2}),
144+
])
145+
monkeypatch.setenv("TINA4_KAFKA_GROUP_ID", "tina4_tomb_drain_" + secrets.token_hex(8))
146+
queue = Queue(topic=topic, backend="kafka")
147+
try:
148+
got = []
149+
for job in queue.consume(topic, poll_interval=0):
150+
got.append(job.data.get("n"))
151+
job.complete()
152+
assert got == [1, 2], f"the drain stopped at the tombstone: {got}"
153+
finally:
154+
queue.close()
155+
156+
157+
def test_raw_path_reads_past_a_tombstone():
158+
"""The socket path steps over a tombstone instead of re-fetching it for ever.
159+
160+
_decode_first_record answered a null value with None, so dequeue() returned
161+
None without moving its offset: every later fetch asked for the tombstone
162+
again, and the record after it was never delivered. No exception, so nothing
163+
said the consumer had stopped.
164+
"""
165+
_require_broker()
166+
topic = _produce([
167+
(b"k1", {"n": 1}),
168+
(b"k1", None),
169+
(b"k2", {"n": 2}),
170+
], one_per_batch=True)
171+
reader = _raw_connector("tina4_tomb_raw_" + secrets.token_hex(8))
172+
try:
173+
got = [reader.dequeue_record(topic), reader.dequeue_record(topic)]
174+
assert [m and m.get("n") for m, _ in got] == [1, 2], got
175+
# The key names the delivered record's own offset, not the tombstone's.
176+
assert [key for _, key in got] == [f"{topic}:0:0", f"{topic}:0:2"], got
177+
finally:
178+
reader.close()
179+
180+
181+
def test_raw_path_still_waits_out_its_deadline_for_a_record_after_tombstones():
182+
"""Past a tombstone the fetch still long-polls, only for the time left.
183+
184+
Capping the broker's wait must not turn it into no wait: a record produced
185+
while the capped fetch waits is delivered by that same dequeue().
186+
"""
187+
import threading
188+
import time
189+
190+
_require_broker()
191+
from confluent_kafka import Producer
192+
193+
topic = _produce([(b"k0", {"n": 0}), (b"k1", None), (b"k1", None)], one_per_batch=True)
194+
reader = _raw_connector("tina4_tomb_rawwait_" + secrets.token_hex(8))
195+
try:
196+
assert (reader.dequeue(topic) or {}).get("n") == 0
197+
# Connected before the wait starts, so only the send lands inside it.
198+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
199+
producer.list_topics(topic, timeout=15)
200+
def late():
201+
time.sleep(0.4)
202+
producer.produce(topic, value=json.dumps({"n": "late"}).encode(), partition=0)
203+
producer.flush(15)
204+
sender = threading.Thread(target=late, daemon=True)
205+
sender.start()
206+
started = time.monotonic()
207+
message, key = reader.dequeue_record(topic)
208+
took = time.monotonic() - started
209+
sender.join(20)
210+
assert message == {"n": "late"}, f"the capped fetch did not wait for the record: {message!r}"
211+
assert key == f"{topic}:0:3", key
212+
assert took < 1.6, f"dequeue() took {took:.1f}s (deadline ~1s)"
213+
finally:
214+
reader.close()
215+
216+
217+
def test_raw_path_cannot_spin_on_a_tombstone_it_cannot_step_past():
218+
"""A tombstone batched ahead of a record must not loop the socket path.
219+
220+
The fetch hands back the whole batch and the raw decoder reads only its
221+
first record, so the tombstone comes back at an offset the reader already
222+
passed. The skip loop must return at once rather than fetch the same offset
223+
until the deadline. (The record after it is still not delivered by the raw
224+
path: that is the separate batch-decoding limit, not covered here.)
225+
"""
226+
import threading
227+
import time
228+
229+
_require_broker()
230+
topic = _produce([(b"k1", None), (b"k2", {"n": 1})]) # one batch
231+
reader = _raw_connector("tina4_tomb_spin_" + secrets.token_hex(8))
232+
took = []
233+
def run():
234+
for _ in range(2):
235+
started = time.monotonic()
236+
reader.dequeue(topic)
237+
took.append(time.monotonic() - started)
238+
worker = threading.Thread(target=run, daemon=True)
239+
try:
240+
worker.start()
241+
worker.join(30)
242+
assert not worker.is_alive(), "dequeue() spun on the tombstone for 30s"
243+
# The first call includes the metadata round trip; the second must not
244+
# sit re-fetching the same offset for the whole ~1 s deadline.
245+
assert took[1] < 0.5, f"dequeue() re-fetched the same offset for {took[1]:.2f}s"
246+
finally:
247+
if not worker.is_alive():
248+
reader.close()
249+
250+
251+
def _committed(topic, group_id):
252+
"""The group's durably committed offset for partition 0 (negative = none)."""
253+
from confluent_kafka import Consumer, TopicPartition
254+
255+
probe = Consumer({"bootstrap.servers": KAFKA_BROKERS, "group.id": group_id,
256+
"enable.auto.commit": False})
257+
try:
258+
return probe.committed([TopicPartition(topic, 0)], timeout=15)[0].offset
259+
finally:
260+
probe.close()
261+
262+
263+
@pytest.mark.parametrize("client", ["confluent", "raw"])
264+
def test_dequeue_keeps_its_deadline_under_a_stream_of_tombstones(client):
265+
"""Skipping tombstones must not hold dequeue() past its ~1 s deadline.
266+
267+
The skip went straight back to poll and never reached the deadline check, so
268+
while tombstones kept arriving (a burst of CDC deletes) one dequeue() blocked
269+
for as long as the stream lasted: 8 s for an 8 s stream.
270+
"""
271+
import threading
272+
import time
273+
274+
_require_broker()
275+
from confluent_kafka import Producer
276+
277+
topic = _produce([(b"k0", {"n": 0})])
278+
group = "tina4_tomb_deadline_" + secrets.token_hex(8)
279+
reader = (KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
280+
if client == "confluent" else _raw_connector(group))
281+
try:
282+
assert (reader.dequeue(topic) or {}).get("n") == 0 # warm: joined + assigned
283+
stop = time.monotonic() + 6
284+
def feed():
285+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
286+
while time.monotonic() < stop:
287+
producer.produce(topic, value=None, partition=0)
288+
producer.flush(5)
289+
time.sleep(0.1)
290+
threading.Thread(target=feed, daemon=True).start()
291+
time.sleep(0.3)
292+
started = time.monotonic()
293+
outcome = []
294+
worker = threading.Thread(target=lambda: outcome.append(reader.dequeue(topic)), daemon=True)
295+
worker.start()
296+
worker.join(20)
297+
took = time.monotonic() - started
298+
assert not worker.is_alive(), "dequeue() never returned under the tombstone stream"
299+
assert outcome == [None]
300+
assert took < 1.6, f"dequeue() took {took:.1f}s under a tombstone stream (deadline ~1s)"
301+
finally:
302+
reader.close()
303+
304+
305+
def test_raw_path_keeps_its_deadline_when_the_tombstones_end_the_topic():
306+
"""Past a tombstone the broker's long-poll wait is held to the deadline too.
307+
308+
The fetch asks the broker to wait up to 5 s for data. After stepping past
309+
tombstones to the end of the topic, the next fetch sat out the whole 5 s,
310+
so dequeue() took 5 s instead of its ~1 s.
311+
"""
312+
import time
313+
314+
_require_broker()
315+
topic = _produce([(b"k", {"n": 0})])
316+
reader = _raw_connector("tina4_tomb_rawtail_" + secrets.token_hex(8))
317+
try:
318+
assert (reader.dequeue(topic) or {}).get("n") == 0
319+
from confluent_kafka import Producer
320+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
321+
for _ in range(3):
322+
producer.produce(topic, value=None, partition=0)
323+
producer.flush(15)
324+
started = time.monotonic()
325+
assert reader.dequeue(topic) is None
326+
took = time.monotonic() - started
327+
assert took < 1.6, f"dequeue() took {took:.1f}s past the tombstones (deadline ~1s)"
328+
# The offset moved past them, so the next call does not rescan them.
329+
assert reader._offsets[topic] == 4
330+
finally:
331+
reader.close()
332+
333+
334+
def test_raw_path_steps_over_an_empty_value():
335+
"""The raw decoder treats an empty value like a null one and steps past it.
336+
337+
Both used to stop the reader on the spot for ever. (The confluent path is
338+
unchanged for an empty value: it returns no message and moves on.)
339+
"""
340+
_require_broker()
341+
from confluent_kafka import Producer
342+
343+
topic = "tina4_tombstone_" + secrets.token_hex(6)
344+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS})
345+
producer.produce(topic, value=b"", partition=0)
346+
producer.flush(15)
347+
producer.produce(topic, value=json.dumps({"n": 1}).encode(), partition=0)
348+
producer.flush(15)
349+
reader = _raw_connector("tina4_tomb_empty_" + secrets.token_hex(8))
350+
try:
351+
assert (reader.dequeue(topic) or {}).get("n") == 1
352+
finally:
353+
reader.close()
354+
355+
356+
def test_raw_path_keeps_its_deadline_through_a_backlog_of_tombstones():
357+
"""A backlog of tombstones longer than one deadline is cut off at it.
358+
359+
Each fetch of a tombstone returns at once, so only the deadline check stops
360+
the loop: without it one dequeue() walked the whole backlog.
361+
"""
362+
import time
363+
364+
from confluent_kafka import Producer
365+
366+
_require_broker()
367+
topic = "tina4_tombstone_" + secrets.token_hex(6)
368+
# One record per batch, so the raw decoder can step over every one.
369+
producer = Producer({"bootstrap.servers": KAFKA_BROKERS,
370+
"batch.num.messages": 1, "linger.ms": 0})
371+
total = 6000
372+
for i in range(total):
373+
producer.produce(topic, value=None, partition=0)
374+
if i % 500 == 0:
375+
producer.poll(0)
376+
producer.flush(60)
377+
reader = _raw_connector("tina4_tomb_backlog_" + secrets.token_hex(8))
378+
try:
379+
started = time.monotonic()
380+
assert reader.dequeue(topic) is None
381+
took = time.monotonic() - started
382+
assert took < 1.6, f"dequeue() took {took:.1f}s on a tombstone backlog (deadline ~1s)"
383+
assert 0 < reader._offsets[topic] < total, reader._offsets
384+
finally:
385+
reader.close()
386+
387+
388+
def test_a_fresh_process_gets_past_a_tombstone_through_the_normal_ack():
389+
"""Nothing commits on the tombstone path; acknowledging the record after it
390+
commits past both, so a restart in the same group neither stops at the
391+
tombstone nor redelivers anything."""
392+
_require_broker()
393+
topic = _produce([(b"k1", {"n": 1}), (b"k1", None), (b"k2", {"n": 2})])
394+
group = "tina4_tomb_restart_" + secrets.token_hex(8)
395+
first = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
396+
try:
397+
message, key = first.dequeue_record(topic)
398+
assert message == {"n": 1}
399+
first.acknowledge(topic, key)
400+
assert _committed(topic, group) == 1 # at the tombstone
401+
finally:
402+
first.close()
403+
second = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
404+
try:
405+
message, key = second.dequeue_record(topic)
406+
assert message == {"n": 2}, f"a restart did not get past the tombstone: {message!r}"
407+
second.acknowledge(topic, key)
408+
assert _committed(topic, group) == 3
409+
finally:
410+
second.close()
411+
third = KafkaConnector(brokers=KAFKA_BROKERS, group_id=group)
412+
try:
413+
assert third.dequeue_record(topic) == (None, None)
414+
finally:
415+
third.close()

0 commit comments

Comments
 (0)