@@ -43,20 +43,35 @@ def _require_broker():
4343 pytest .skip ("kafka not set: export TINA4_KAFKA_BROKERS (e.g. 127.0.0.1:9092)" )
4444
4545
46- def _produce (records ):
46+ def _produce (records , one_per_batch = False ):
4747 """Produce (key, value) pairs with the REAL confluent Producer. A value of
48- None is a genuine tombstone on the wire, exactly like a Debezium delete."""
48+ None is a genuine tombstone on the wire, exactly like a Debezium delete.
49+
50+ one_per_batch flushes after every record so each lands in its own record
51+ batch: the raw socket path decodes only the first record of a fetched batch.
52+ """
4953 from confluent_kafka import Producer
5054
5155 topic = "tina4_tombstone_" + secrets .token_hex (6 )
5256 producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS })
5357 for key , value in records :
5458 payload = None if value is None else json .dumps (value ).encode ("utf-8" )
55- producer .produce (topic , key = key , value = payload )
59+ producer .produce (topic , key = key , value = payload , partition = 0 )
60+ if one_per_batch :
61+ producer .flush (15 )
5662 producer .flush (15 )
5763 return topic
5864
5965
66+ def _raw_connector (group_id ):
67+ """The zero-dependency socket path, selected the way TestKafkaRawPathLive
68+ does: the flag the constructor sets from its import probe."""
69+ reader = KafkaConnector (brokers = KAFKA_BROKERS , group_id = group_id )
70+ reader ._use_confluent = False
71+ reader ._confluent = None
72+ return reader
73+
74+
6075def _drain (reader , topic , want_key , max_calls = 12 ):
6176 """Call dequeue up to max_calls times, returning the first message whose
6277 'event' equals want_key (or None). Never swallows an exception -- a crash in
@@ -109,3 +124,292 @@ def test_only_a_tombstone_yields_no_job_and_no_crash():
109124 assert reader .dequeue (topic ) is None
110125 finally :
111126 reader .close ()
127+
128+
129+ def test_single_pass_drain_reads_past_a_tombstone (monkeypatch ):
130+ """consume(poll_interval=0) drains to the end of the topic, tombstone or not.
131+
132+ Answering a tombstone with "no message" reads as an empty queue, so a
133+ single-pass drain stopped at it and left every later record behind; a fresh
134+ process (a cron drain) resumed at the tombstone's committed offset and
135+ stopped there again, every run.
136+ """
137+ _require_broker ()
138+ from tina4_python .queue import Queue
139+
140+ topic = _produce ([
141+ (b"k1" , {"n" : 1 }),
142+ (b"k1" , None ),
143+ (b"k2" , {"n" : 2 }),
144+ ])
145+ monkeypatch .setenv ("TINA4_KAFKA_GROUP_ID" , "tina4_tomb_drain_" + secrets .token_hex (8 ))
146+ queue = Queue (topic = topic , backend = "kafka" )
147+ try :
148+ got = []
149+ for job in queue .consume (topic , poll_interval = 0 ):
150+ got .append (job .data .get ("n" ))
151+ job .complete ()
152+ assert got == [1 , 2 ], f"the drain stopped at the tombstone: { got } "
153+ finally :
154+ queue .close ()
155+
156+
157+ def test_raw_path_reads_past_a_tombstone ():
158+ """The socket path steps over a tombstone instead of re-fetching it for ever.
159+
160+ _decode_first_record answered a null value with None, so dequeue() returned
161+ None without moving its offset: every later fetch asked for the tombstone
162+ again, and the record after it was never delivered. No exception, so nothing
163+ said the consumer had stopped.
164+ """
165+ _require_broker ()
166+ topic = _produce ([
167+ (b"k1" , {"n" : 1 }),
168+ (b"k1" , None ),
169+ (b"k2" , {"n" : 2 }),
170+ ], one_per_batch = True )
171+ reader = _raw_connector ("tina4_tomb_raw_" + secrets .token_hex (8 ))
172+ try :
173+ got = [reader .dequeue_record (topic ), reader .dequeue_record (topic )]
174+ assert [m and m .get ("n" ) for m , _ in got ] == [1 , 2 ], got
175+ # The key names the delivered record's own offset, not the tombstone's.
176+ assert [key for _ , key in got ] == [f"{ topic } :0:0" , f"{ topic } :0:2" ], got
177+ finally :
178+ reader .close ()
179+
180+
181+ def test_raw_path_still_waits_out_its_deadline_for_a_record_after_tombstones ():
182+ """Past a tombstone the fetch still long-polls, only for the time left.
183+
184+ Capping the broker's wait must not turn it into no wait: a record produced
185+ while the capped fetch waits is delivered by that same dequeue().
186+ """
187+ import threading
188+ import time
189+
190+ _require_broker ()
191+ from confluent_kafka import Producer
192+
193+ topic = _produce ([(b"k0" , {"n" : 0 }), (b"k1" , None ), (b"k1" , None )], one_per_batch = True )
194+ reader = _raw_connector ("tina4_tomb_rawwait_" + secrets .token_hex (8 ))
195+ try :
196+ assert (reader .dequeue (topic ) or {}).get ("n" ) == 0
197+ # Connected before the wait starts, so only the send lands inside it.
198+ producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS })
199+ producer .list_topics (topic , timeout = 15 )
200+ def late ():
201+ time .sleep (0.4 )
202+ producer .produce (topic , value = json .dumps ({"n" : "late" }).encode (), partition = 0 )
203+ producer .flush (15 )
204+ sender = threading .Thread (target = late , daemon = True )
205+ sender .start ()
206+ started = time .monotonic ()
207+ message , key = reader .dequeue_record (topic )
208+ took = time .monotonic () - started
209+ sender .join (20 )
210+ assert message == {"n" : "late" }, f"the capped fetch did not wait for the record: { message !r} "
211+ assert key == f"{ topic } :0:3" , key
212+ assert took < 1.6 , f"dequeue() took { took :.1f} s (deadline ~1s)"
213+ finally :
214+ reader .close ()
215+
216+
217+ def test_raw_path_cannot_spin_on_a_tombstone_it_cannot_step_past ():
218+ """A tombstone batched ahead of a record must not loop the socket path.
219+
220+ The fetch hands back the whole batch and the raw decoder reads only its
221+ first record, so the tombstone comes back at an offset the reader already
222+ passed. The skip loop must return at once rather than fetch the same offset
223+ until the deadline. (The record after it is still not delivered by the raw
224+ path: that is the separate batch-decoding limit, not covered here.)
225+ """
226+ import threading
227+ import time
228+
229+ _require_broker ()
230+ topic = _produce ([(b"k1" , None ), (b"k2" , {"n" : 1 })]) # one batch
231+ reader = _raw_connector ("tina4_tomb_spin_" + secrets .token_hex (8 ))
232+ took = []
233+ def run ():
234+ for _ in range (2 ):
235+ started = time .monotonic ()
236+ reader .dequeue (topic )
237+ took .append (time .monotonic () - started )
238+ worker = threading .Thread (target = run , daemon = True )
239+ try :
240+ worker .start ()
241+ worker .join (30 )
242+ assert not worker .is_alive (), "dequeue() spun on the tombstone for 30s"
243+ # The first call includes the metadata round trip; the second must not
244+ # sit re-fetching the same offset for the whole ~1 s deadline.
245+ assert took [1 ] < 0.5 , f"dequeue() re-fetched the same offset for { took [1 ]:.2f} s"
246+ finally :
247+ if not worker .is_alive ():
248+ reader .close ()
249+
250+
251+ def _committed (topic , group_id ):
252+ """The group's durably committed offset for partition 0 (negative = none)."""
253+ from confluent_kafka import Consumer , TopicPartition
254+
255+ probe = Consumer ({"bootstrap.servers" : KAFKA_BROKERS , "group.id" : group_id ,
256+ "enable.auto.commit" : False })
257+ try :
258+ return probe .committed ([TopicPartition (topic , 0 )], timeout = 15 )[0 ].offset
259+ finally :
260+ probe .close ()
261+
262+
263+ @pytest .mark .parametrize ("client" , ["confluent" , "raw" ])
264+ def test_dequeue_keeps_its_deadline_under_a_stream_of_tombstones (client ):
265+ """Skipping tombstones must not hold dequeue() past its ~1 s deadline.
266+
267+ The skip went straight back to poll and never reached the deadline check, so
268+ while tombstones kept arriving (a burst of CDC deletes) one dequeue() blocked
269+ for as long as the stream lasted: 8 s for an 8 s stream.
270+ """
271+ import threading
272+ import time
273+
274+ _require_broker ()
275+ from confluent_kafka import Producer
276+
277+ topic = _produce ([(b"k0" , {"n" : 0 })])
278+ group = "tina4_tomb_deadline_" + secrets .token_hex (8 )
279+ reader = (KafkaConnector (brokers = KAFKA_BROKERS , group_id = group )
280+ if client == "confluent" else _raw_connector (group ))
281+ try :
282+ assert (reader .dequeue (topic ) or {}).get ("n" ) == 0 # warm: joined + assigned
283+ stop = time .monotonic () + 6
284+ def feed ():
285+ producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS })
286+ while time .monotonic () < stop :
287+ producer .produce (topic , value = None , partition = 0 )
288+ producer .flush (5 )
289+ time .sleep (0.1 )
290+ threading .Thread (target = feed , daemon = True ).start ()
291+ time .sleep (0.3 )
292+ started = time .monotonic ()
293+ outcome = []
294+ worker = threading .Thread (target = lambda : outcome .append (reader .dequeue (topic )), daemon = True )
295+ worker .start ()
296+ worker .join (20 )
297+ took = time .monotonic () - started
298+ assert not worker .is_alive (), "dequeue() never returned under the tombstone stream"
299+ assert outcome == [None ]
300+ assert took < 1.6 , f"dequeue() took { took :.1f} s under a tombstone stream (deadline ~1s)"
301+ finally :
302+ reader .close ()
303+
304+
305+ def test_raw_path_keeps_its_deadline_when_the_tombstones_end_the_topic ():
306+ """Past a tombstone the broker's long-poll wait is held to the deadline too.
307+
308+ The fetch asks the broker to wait up to 5 s for data. After stepping past
309+ tombstones to the end of the topic, the next fetch sat out the whole 5 s,
310+ so dequeue() took 5 s instead of its ~1 s.
311+ """
312+ import time
313+
314+ _require_broker ()
315+ topic = _produce ([(b"k" , {"n" : 0 })])
316+ reader = _raw_connector ("tina4_tomb_rawtail_" + secrets .token_hex (8 ))
317+ try :
318+ assert (reader .dequeue (topic ) or {}).get ("n" ) == 0
319+ from confluent_kafka import Producer
320+ producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS })
321+ for _ in range (3 ):
322+ producer .produce (topic , value = None , partition = 0 )
323+ producer .flush (15 )
324+ started = time .monotonic ()
325+ assert reader .dequeue (topic ) is None
326+ took = time .monotonic () - started
327+ assert took < 1.6 , f"dequeue() took { took :.1f} s past the tombstones (deadline ~1s)"
328+ # The offset moved past them, so the next call does not rescan them.
329+ assert reader ._offsets [topic ] == 4
330+ finally :
331+ reader .close ()
332+
333+
334+ def test_raw_path_steps_over_an_empty_value ():
335+ """The raw decoder treats an empty value like a null one and steps past it.
336+
337+ Both used to stop the reader on the spot for ever. (The confluent path is
338+ unchanged for an empty value: it returns no message and moves on.)
339+ """
340+ _require_broker ()
341+ from confluent_kafka import Producer
342+
343+ topic = "tina4_tombstone_" + secrets .token_hex (6 )
344+ producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS })
345+ producer .produce (topic , value = b"" , partition = 0 )
346+ producer .flush (15 )
347+ producer .produce (topic , value = json .dumps ({"n" : 1 }).encode (), partition = 0 )
348+ producer .flush (15 )
349+ reader = _raw_connector ("tina4_tomb_empty_" + secrets .token_hex (8 ))
350+ try :
351+ assert (reader .dequeue (topic ) or {}).get ("n" ) == 1
352+ finally :
353+ reader .close ()
354+
355+
356+ def test_raw_path_keeps_its_deadline_through_a_backlog_of_tombstones ():
357+ """A backlog of tombstones longer than one deadline is cut off at it.
358+
359+ Each fetch of a tombstone returns at once, so only the deadline check stops
360+ the loop: without it one dequeue() walked the whole backlog.
361+ """
362+ import time
363+
364+ from confluent_kafka import Producer
365+
366+ _require_broker ()
367+ topic = "tina4_tombstone_" + secrets .token_hex (6 )
368+ # One record per batch, so the raw decoder can step over every one.
369+ producer = Producer ({"bootstrap.servers" : KAFKA_BROKERS ,
370+ "batch.num.messages" : 1 , "linger.ms" : 0 })
371+ total = 6000
372+ for i in range (total ):
373+ producer .produce (topic , value = None , partition = 0 )
374+ if i % 500 == 0 :
375+ producer .poll (0 )
376+ producer .flush (60 )
377+ reader = _raw_connector ("tina4_tomb_backlog_" + secrets .token_hex (8 ))
378+ try :
379+ started = time .monotonic ()
380+ assert reader .dequeue (topic ) is None
381+ took = time .monotonic () - started
382+ assert took < 1.6 , f"dequeue() took { took :.1f} s on a tombstone backlog (deadline ~1s)"
383+ assert 0 < reader ._offsets [topic ] < total , reader ._offsets
384+ finally :
385+ reader .close ()
386+
387+
388+ def test_a_fresh_process_gets_past_a_tombstone_through_the_normal_ack ():
389+ """Nothing commits on the tombstone path; acknowledging the record after it
390+ commits past both, so a restart in the same group neither stops at the
391+ tombstone nor redelivers anything."""
392+ _require_broker ()
393+ topic = _produce ([(b"k1" , {"n" : 1 }), (b"k1" , None ), (b"k2" , {"n" : 2 })])
394+ group = "tina4_tomb_restart_" + secrets .token_hex (8 )
395+ first = KafkaConnector (brokers = KAFKA_BROKERS , group_id = group )
396+ try :
397+ message , key = first .dequeue_record (topic )
398+ assert message == {"n" : 1 }
399+ first .acknowledge (topic , key )
400+ assert _committed (topic , group ) == 1 # at the tombstone
401+ finally :
402+ first .close ()
403+ second = KafkaConnector (brokers = KAFKA_BROKERS , group_id = group )
404+ try :
405+ message , key = second .dequeue_record (topic )
406+ assert message == {"n" : 2 }, f"a restart did not get past the tombstone: { message !r} "
407+ second .acknowledge (topic , key )
408+ assert _committed (topic , group ) == 3
409+ finally :
410+ second .close ()
411+ third = KafkaConnector (brokers = KAFKA_BROKERS , group_id = group )
412+ try :
413+ assert third .dequeue_record (topic ) == (None , None )
414+ finally :
415+ third .close ()
0 commit comments