diff --git a/src/services/service-handler-base.ts b/src/services/service-handler-base.ts index 046c9002..b5a47068 100644 --- a/src/services/service-handler-base.ts +++ b/src/services/service-handler-base.ts @@ -153,7 +153,11 @@ export abstract class ServiceHandlerBase { }, }) .eq('id', (candidate as any).id) - .eq('status', 'escrowed') // optimistic concurrency — lost race ⇒ 0 rows + // NOT exclusive: this writes metadata and leaves status 'escrowed', so a second claimer racing this + // one also matches and also gets the row (until 2026-11-16 this said "lost race ⇒ 0 rows"). It + // only stops a claim of a row that already moved on. Exactly-once is the guarded escrowed → + // fulfilled transition in processOne. + .eq('status', 'escrowed') .select() .maybeSingle(); @@ -263,6 +267,41 @@ export abstract class ServiceHandlerBase { : undefined; if (routesToDispute(result as any)) { + // ONLY FROM escrowed, and only the run that makes the transition files the dispute (Strix on + // #1560). Both runs used to insert a dispute_validation_queue row and the dispute worker applies + // deltas per row, so one contract was charged twice. Transition first, then file. + // NB: do NOT write dispute_panel_validation_queue_id here — that + // column's FK references validation_queue(id), NOT + // dispute_validation_queue. The contract↔dispute link is preserved + // by dispute_validation_queue.contract_id (set below). + const { data: disputed, error: dispErr } = await db + .from('service_contracts') + .update({ + status: 'disputed', + result, + disputed_at: new Date().toISOString(), + }) + .eq('id', contract.id) + .eq('status', 'escrowed') + .select('id') + .maybeSingle(); + + if (dispErr) { + console.error( + `[${this.serviceType}] disputed-transition failed for contract ${contract.id}:`, + dispErr?.message ?? dispErr, + (dispErr as any)?.stack ?? new Error().stack + ); + throw new Error(`Disputed status transition failed: ${dispErr.message}`); + } + if (!disputed) { + console.warn( + `[${this.serviceType}] contract ${contract.id} was no longer escrowed at the disputed transition: ` + + 'another run got there first. No second dispute filed.' + ); + return { processed: false, contract_id: contract.id }; + } + const { data: dq, error: dqErr } = await db .from('dispute_validation_queue') .insert({ @@ -290,33 +329,24 @@ export abstract class ServiceHandlerBase { dqErr?.message ?? dqErr, (dqErr as any)?.stack ?? new Error().stack ); + // Put the contract back so a later cycle retries; a disputed contract with no queue row would + // never be resolved. If the put-back fails too, the contract is visibly stuck, never paid twice. + const { error: backErr } = await db + .from('service_contracts') + .update({ status: 'escrowed', result: (contract as any).result ?? null, disputed_at: null }) + .eq('id', contract.id) + .eq('status', 'disputed'); + if (backErr) { + console.error( + `[${this.serviceType}] contract ${contract.id} is disputed with NO dispute row, and could not be put back to escrowed:`, + backErr?.message ?? backErr + ); + } throw new Error( `dispute_validation_queue insert failed: ${dqErr?.message ?? 'no row returned'}` ); } - // NB: do NOT write dispute_panel_validation_queue_id here — that - // column's FK references validation_queue(id), NOT - // dispute_validation_queue. The contract↔dispute link is preserved - // by dispute_validation_queue.contract_id (set above). - const { error: dispErr } = await db - .from('service_contracts') - .update({ - status: 'disputed', - result, - disputed_at: new Date().toISOString(), - }) - .eq('id', contract.id); - - if (dispErr) { - console.error( - `[${this.serviceType}] disputed-transition failed for contract ${contract.id}:`, - dispErr?.message ?? dispErr, - (dispErr as any)?.stack ?? new Error().stack - ); - throw new Error(`Disputed status transition failed: ${dispErr.message}`); - } - console.log( `[${this.serviceType}] contract ${contract.id} verdict=${verdict} ` + `→ routed to dispute_validation_queue ${(dq as any).id} ` + @@ -326,14 +356,22 @@ export abstract class ServiceHandlerBase { } // PASS / confident FAIL / UNDETERMINED / verdict-less handler → fulfilled path. - const { error: updateErr } = await db + // ONLY FROM escrowed, and only the run that makes the transition applies the deltas below. + // The claim above does not change status, so two runs can both hold one contract; until + // 2026-11-16 this update had no status guard and both ran the economy. MEASURED: contract + // 7ca47a1d (2026-07-08) got two SERVICE_FULFILLED rounds 3.5 s apart, two outbox rows and + // three on-chain writes; the buyer took +5 twice. POST /contracts/:id/fulfill already guards. + const { data: transitioned, error: updateErr } = await db .from('service_contracts') .update({ status: 'fulfilled', result, fulfilled_at: new Date().toISOString(), }) - .eq('id', contract.id); + .eq('id', contract.id) + .eq('status', 'escrowed') + .select('id') + .maybeSingle(); if (updateErr) { console.error( @@ -343,6 +381,14 @@ export abstract class ServiceHandlerBase { ); throw new Error(`Status transition failed: ${updateErr.message}`); } + if (!transitioned) { + // Another run fulfilled (or disputed) it first. That run applied the deltas; this one must not. + console.warn( + `[${this.serviceType}] contract ${contract.id} was no longer escrowed at the fulfilled transition: ` + + 'another run got there first. SERVICE_FULFILLED deltas NOT applied a second time.' + ); + return { processed: false, contract_id: contract.id }; + } // Canonical Phase 2.6 economy. Signature (Phase 2.9): // applyServiceFulfilledDeltas({id, service_id, provider_agent_id, buyer_agent_id}) diff --git a/tests/service-handler-fulfil-once.test.ts b/tests/service-handler-fulfil-once.test.ts new file mode 100644 index 00000000..bdb5767a --- /dev/null +++ b/tests/service-handler-fulfil-once.test.ts @@ -0,0 +1,143 @@ +/** + * One contract, one SERVICE_FULFILLED round, however many runs hold it (2026-11-16). + * + * MEASURED: contract 7ca47a1d (2026-07-08) went through applyServiceFulfilledDeltas twice, 3.5 s + * apart. Two outbox rows, three on-chain writes, and the buyer took +5 twice (1030→1035→1040). The + * claim leaves status 'escrowed', so two concurrent runs both hold the contract, and the + * escrowed → fulfilled update had no status guard, so both ran the economy. + * + * The fake table below honours the update's filters, the way PostgREST does: an update whose + * `.eq('status','escrowed')` no longer matches changes nothing and returns no row. + */ +process.env['SUPABASE_URL'] = process.env['SUPABASE_URL'] || 'http://localhost:54321'; +process.env['SUPABASE_SERVICE_KEY'] = process.env['SUPABASE_SERVICE_KEY'] || 'dummy'; + +const dbFrom = jest.fn(); +const applyDeltas = jest.fn(async () => undefined); +jest.mock('../src/db', () => ({ db: { from: (...a: unknown[]) => dbFrom(...a) } })); +jest.mock('../src/services/validation-repid-delta', () => ({ applyServiceFulfilledDeltas: (...a: unknown[]) => (applyDeltas as any)(...a) })); +jest.mock('../src/services/service-quality-hook', () => ({ + recordServiceQuality: jest.fn(async () => ({ mode: 'off', checked: false, reason: 'hook_disabled', observed_at: '' })), +})); + +import { ServiceHandlerBase } from '../src/services/service-handler-base'; + +type Row = Record; +let row: Row; +const disputeRows: Row[] = []; + +function table() { + dbFrom.mockImplementation((name: string) => { + if (name === 'dispute_validation_queue') { + const q: any = { + insert: (r: Row) => { const added = { ...r, id: `dq-${disputeRows.length + 1}` }; disputeRows.push(added); q.added = added; return q; }, + select: () => q, + single: async () => ({ data: q.added, error: null }), + }; + return q; + } + if (name !== 'service_contracts') throw new Error(`unexpected table ${name}`); + const filters: Array<[string, unknown]> = []; + let patch: Row | null = null; + const matches = () => filters.every(([k, v]) => k.includes('.') || row[k] === v); + const run = () => { + if (!matches()) return null; + if (patch) Object.assign(row, patch); + return { ...row }; + }; + const q: any = { + select: () => q, + update: (p: Row) => { patch = p; return q; }, + eq: (k: string, v: unknown) => { filters.push([k, v]); return q; }, + not: () => q, + order: () => q, + limit: () => q, + maybeSingle: async () => ({ data: run(), error: null }), + then: (ok: (v: unknown) => unknown, bad?: (e: unknown) => unknown) => Promise.resolve({ data: null, error: null, _: run() }).then(ok, bad), + }; + return q; + }); +} + +let release!: () => void; +let gate: Promise; + +let verdict: Record = { verdict: 'PASS' }; + +class GatedHandler extends ServiceHandlerBase { + protected readonly serviceType = 'verification'; + protected async fulfill(): Promise> { + await gate; // both runs have claimed before either transitions + return verdict; + } +} + +beforeEach(() => { + applyDeltas.mockClear(); + disputeRows.length = 0; + verdict = { verdict: 'PASS' }; + row = { + id: 'c-1', status: 'escrowed', work_statement_hash: '0x' + 'ab'.repeat(32), created_at: '2026-07-08T20:17:31Z', + provider_agent_id: 'p-1', buyer_agent_id: 'b-1', service_id: 's-1', metadata: {}, + }; + gate = new Promise((r) => (release = r)); + table(); +}); + +test('two concurrent runs on one contract: both claim, ONE fulfils and applies the deltas', async () => { + const h = new GatedHandler(); + const a = h.processOne('p-1'); + const b = h.processOne('p-1'); + await new Promise((r) => setImmediate(r)); + release(); + const [ra, rb] = await Promise.all([a, b]); + expect(row.status).toBe('fulfilled'); + expect(applyDeltas).toHaveBeenCalledTimes(1); + expect([ra.processed, rb.processed].sort()).toEqual([false, true]); +}); + +test('a single run still fulfils and applies the deltas once (control)', async () => { + release(); + const out = await new GatedHandler().processOne('p-1'); + expect(out).toEqual({ processed: true, contract_id: 'c-1' }); + expect(applyDeltas).toHaveBeenCalledTimes(1); +}); + +test('a contract that moved on (e.g. disputed by another run) is not fulfilled and earns nothing', async () => { + const h = new GatedHandler(); + const p = h.processOne('p-1'); + await new Promise((r) => setImmediate(r)); + row.status = 'disputed'; + release(); + const out = await p; + expect(out.processed).toBe(false); + expect(row.status).toBe('disputed'); + expect(applyDeltas).not.toHaveBeenCalled(); +}); + +test('two concurrent runs that both route to DISPUTE file ONE dispute row (Strix on #1560)', async () => { + verdict = { verdict: 'VETO' }; + const h = new GatedHandler(); + const a = h.processOne('p-1'); + const b = h.processOne('p-1'); + await new Promise((r) => setImmediate(r)); + release(); + const [ra, rb] = await Promise.all([a, b]); + expect(row.status).toBe('disputed'); + expect(disputeRows).toHaveLength(1); + expect([ra.processed, rb.processed].sort()).toEqual([false, true]); + expect(applyDeltas).not.toHaveBeenCalled(); +}); + +test('a contract another run already FULFILLED is not flipped to disputed, and files no dispute', async () => { + verdict = { verdict: 'VETO' }; + const p = new GatedHandler().processOne('p-1'); + await new Promise((r) => setImmediate(r)); + row.status = 'fulfilled'; + release(); + const out = await p; + expect(out.processed).toBe(false); + expect(row.status).toBe('fulfilled'); + expect(disputeRows).toHaveLength(0); +}); + diff --git a/tests/services/service-handler-base.test.ts b/tests/services/service-handler-base.test.ts index 10f90457..fa147fa3 100644 --- a/tests/services/service-handler-base.test.ts +++ b/tests/services/service-handler-base.test.ts @@ -67,6 +67,7 @@ describe('ServiceHandlerBase (Phase 2.10 production superset)', () => { it('processOne claims, fulfils, applies SERVICE_FULFILLED deltas', async () => { maybeSingleQueue.push({ data: contract, error: null }); // fetch candidate maybeSingleQueue.push({ data: contract, error: null }); // claim wins + maybeSingleQueue.push({ data: { id: 'c1' }, error: null }); // escrowed -> fulfilled transition wins const h = new TestHandler(); const r = await h.processOne('p1'); expect(r).toEqual({ processed: true, contract_id: 'c1' }); @@ -95,7 +96,8 @@ describe('ServiceHandlerBase: no SERVICE_FULFILLED deltas for an unstaffed check protected async fulfill(): Promise> { return this.out; } } const run = async (out: Record) => { - maybeSingleQueue.push({ data: contract, error: null }, { data: contract, error: null }); + // fetch candidate, claim wins, escrowed -> fulfilled transition wins + maybeSingleQueue.push({ data: contract, error: null }, { data: contract, error: null }, { data: { id: 'c1' }, error: null }); return new Returns(out).processOne('p1'); }; let log: jest.SpyInstance;