From f1eb0efa2336dd7bdc9c17e4343507aaae051c13 Mon Sep 17 00:00:00 2001 From: mkhullar Date: Tue, 26 May 2026 14:46:04 -0400 Subject: [PATCH] Upsert missing index Signed-off-by: mkhullar --- berkdb/dbinc/db_am.h | 2 +- tests/upsert_missing_index.test/Makefile | 8 ++ tests/upsert_missing_index.test/lrl.options | 2 + tests/upsert_missing_index.test/runit | 130 ++++++++++++++++++++ tests/upsert_missing_index.test/t1.csc2 | 22 ++++ 5 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 tests/upsert_missing_index.test/Makefile create mode 100644 tests/upsert_missing_index.test/lrl.options create mode 100755 tests/upsert_missing_index.test/runit create mode 100644 tests/upsert_missing_index.test/t1.csc2 diff --git a/berkdb/dbinc/db_am.h b/berkdb/dbinc/db_am.h index a4da01ffa0..fc3155480d 100644 --- a/berkdb/dbinc/db_am.h +++ b/berkdb/dbinc/db_am.h @@ -61,7 +61,7 @@ extern __thread DB_LSN commit_lsn; dbenv->rep_log_trigger_cb(log_trigger, lsnp, &commit_lsn, fname, argp->type, argp); \ } \ if (ret != 0) { \ - if (ret == DB_DELETED || ret == DB_IGNORED) { \ + if (ret == DB_IGNORED || (ret == DB_DELETED && IS_RECOVERING(dbenv))) { \ ret = 0; \ goto done; \ } \ diff --git a/tests/upsert_missing_index.test/Makefile b/tests/upsert_missing_index.test/Makefile new file mode 100644 index 0000000000..4f1a0fb623 --- /dev/null +++ b/tests/upsert_missing_index.test/Makefile @@ -0,0 +1,8 @@ +ifeq ($(TESTSROOTDIR),) + include ../testcase.mk +else + include $(TESTSROOTDIR)/testcase.mk +endif +ifeq ($(TEST_TIMEOUT),) + export TEST_TIMEOUT=10m +endif diff --git a/tests/upsert_missing_index.test/lrl.options b/tests/upsert_missing_index.test/lrl.options new file mode 100644 index 0000000000..66bc0fc106 --- /dev/null +++ b/tests/upsert_missing_index.test/lrl.options @@ -0,0 +1,2 @@ +table t1 t1.csc2 +osql_verify_retry_max 1000 diff --git a/tests/upsert_missing_index.test/runit b/tests/upsert_missing_index.test/runit new file mode 100755 index 0000000000..53f0e5be6e --- /dev/null +++ b/tests/upsert_missing_index.test/runit @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +bash -n "$0" | exit 1 + +source ${TESTSROOTDIR}/tools/runit_common.sh +source ${TESTSROOTDIR}/tools/cluster_utils.sh + +set -x + +dbnm=$1 +if [ "x$dbnm" == "x" ] ; then + echo "need a DB name" + exit 1 +fi + +if [[ -z "$CLUSTER" ]]; then + echo "This test requires a cluster, skipping" + exit 0 +fi + +# Uses fault injection (test_dbreg_deleted_intvl) on one replicant to +# simulate a transient DB_DELETED in REC_INTRO during replication apply. +# +# Without the fix: REC_INTRO silently skips the committed btree op, leaving +# a missing index key on the replicant. +# +# With the fix: REC_INTRO does not silently skip DB_DELETED during replication; +# the replicant crashes (crash+recovery is correct - no silent corruption). +# The test verifies surviving nodes have no missing keys and treats the +# expected crash as a pass. + +ITERATIONS=200 +BATCH=10 +NUM_INSERTERS=4 + +master=$(getmaster) +echo "Master is $master" + +# Pick one replicant for injection +for node in $CLUSTER; do + if [[ "$node" != "$master" ]]; then + target=$node + break + fi +done +echo "Enabling test_dbreg_deleted_intvl=3 on $target" +cdb2sql ${CDB2_OPTIONS} --host $target $dbnm \ + "exec procedure sys.cmd.send('test_dbreg_deleted_intvl 3')" + +do_upserts() { + local inserter_id=$1 + local j=0 + while [[ $j -lt $ITERATIONS ]]; do + local sqlfl=$(mktemp) + echo "begin" > $sqlfl + local i=0 + local base=$((inserter_id * ITERATIONS * BATCH + j * BATCH)) + while [[ $i -lt $BATCH ]]; do + local id=$((base + i)) + echo "insert into t1(a,b,c,d) values($id,$id,$((id%3)),$((id%2)))" >> $sqlfl + let i=i+1 + done + echo "commit" >> $sqlfl + cdb2sql ${CDB2_OPTIONS} --host $master $dbnm -f $sqlfl > /dev/null 2>&1 + rm -f $sqlfl + let j=j+1 + done + echo "inserter $inserter_id done" +} + +for k in $(seq 1 $NUM_INSERTERS); do + do_upserts $k & +done +wait + +# Disable injection (auto-disabled after firing, but reset explicitly) +cdb2sql ${CDB2_OPTIONS} --host $target $dbnm \ + "exec procedure sys.cmd.send('test_dbreg_deleted_intvl 0')" 2>/dev/null || true + +sleep 3 + +# Check if target crashed (fix in place) or is still up (no fix) +target_up=1 +cdb2sql ${CDB2_OPTIONS} --host $target $dbnm "select 1" > /dev/null 2>&1 || target_up=0 + +if [[ $target_up -eq 0 ]]; then + echo "$target crashed during replication apply - checking surviving nodes for corruption" + # Target crashed: this is the correct behavior with the fix applied. + # Verify surviving nodes have no missing keys. + found_bug=0 + for node in $CLUSTER; do + [[ "$node" == "$target" ]] && continue + echo "Verify on $node:" + verify_out=$(cdb2sql -s --tabs ${CDB2_OPTIONS} --host $node $dbnm \ + "exec procedure sys.cmd.verify('t1')" 2>&1) + if echo "$verify_out" | grep -q "missing key"; then + echo "BUG: missing key on $node" + found_bug=1 + else + echo "$node: OK" + fi + done + if [[ $found_bug -eq 1 ]]; then + failexit "BUG: missing keys found on surviving nodes after target crash" + fi + # Expected crash with fix applied - remove core so framework doesn't flag it + ssh -o StrictHostKeyChecking=no $target "rm -f ${DBDIR}/core* 2>/dev/null" || true + echo "Success - target crashed as expected (fix working), no corruption on surviving nodes" +else + # Target still up: check all nodes for missing keys + echo "$target still up - checking all nodes for missing keys" + found_bug=0 + for node in $CLUSTER; do + echo "Verify on $node:" + verify_out=$(cdb2sql -s --tabs ${CDB2_OPTIONS} --host $node $dbnm \ + "exec procedure sys.cmd.verify('t1')" 2>&1) + if echo "$verify_out" | grep -q "missing key"; then + echo "BUG: missing key on $node: $(echo "$verify_out" | grep 'missing key' | head -3)" + found_bug=1 + elif echo "$verify_out" | grep -q "Verify failed"; then + echo "BUG: verify failed on $node" + found_bug=1 + else + echo "$node: OK" + fi + done + if [[ $found_bug -eq 1 ]]; then + failexit "BUG REPRODUCED: missing index keys detected (no fix applied)" + fi + echo "Success - no missing keys detected" +fi diff --git a/tests/upsert_missing_index.test/t1.csc2 b/tests/upsert_missing_index.test/t1.csc2 new file mode 100644 index 0000000000..57c9d0a242 --- /dev/null +++ b/tests/upsert_missing_index.test/t1.csc2 @@ -0,0 +1,22 @@ +schema +{ + longlong a + longlong b + int c + int d +} + +keys +{ + /* index 0 - unique on a, upsert conflict target */ + "ix0" = a + + /* index 1 - unique on b+d */ + "ix1" = b + d + + /* index 2 - dup */ + dup "ix2" = b + + /* index 3 - unique on a+c, the index with the missing key */ + "ix3" = a + c +}