diff --git a/apps/server/migrations/0054_source_references.sql b/apps/server/migrations/0054_source_references.sql new file mode 100644 index 00000000..399418bd --- /dev/null +++ b/apps/server/migrations/0054_source_references.sql @@ -0,0 +1,12 @@ +-- Existing Source snapshots need an explicit deployment decision before this upgrade. +CREATE TEMP TABLE source_reference_preflight (empty INTEGER CONSTRAINT legacy_sources_require_explicit_migration CHECK (empty = 1)); +INSERT INTO source_reference_preflight SELECT NOT EXISTS (SELECT 1 FROM event_sources); +DROP TABLE source_reference_preflight; + +ALTER TABLE event_sources ADD COLUMN connector_json TEXT CHECK (connector_json IS NULL OR json_valid(connector_json)); +ALTER TABLE event_sources ADD COLUMN pending_json TEXT CHECK (pending_json IS NULL OR json_valid(pending_json)); +CREATE TABLE event_source_namespace ( + id INTEGER PRIMARY KEY CHECK (id = 1), + value TEXT NOT NULL +) STRICT; +INSERT INTO event_source_namespace VALUES (1, lower(hex(randomblob(16)))); diff --git a/apps/server/node/application/event-source-control.ts b/apps/server/node/application/event-source-control.ts index 074d6510..ee0838ff 100644 --- a/apps/server/node/application/event-source-control.ts +++ b/apps/server/node/application/event-source-control.ts @@ -50,10 +50,10 @@ export class EventSourceControl { return this.eventSources.update(sourceId, input, signal) } - deleteEventSource(actorId: string, sourceId: string, revision: number) { + deleteEventSource(actorId: string, sourceId: string, revision: number, signal = AbortSignal.timeout(30_000)) { this.access.requireAdmin(actorId) if (this.eventSources == null) throw new ControlError(controlErrorCode.eventSourceInvalid, 'Event sources are unavailable.') - this.eventSources.delete(sourceId, revision) + return this.eventSources.delete(sourceId, revision, signal) } } diff --git a/apps/server/node/deployment/connector-contract.ts b/apps/server/node/deployment/connector-contract.ts index 7079772e..f0f3e8f2 100644 --- a/apps/server/node/deployment/connector-contract.ts +++ b/apps/server/node/deployment/connector-contract.ts @@ -78,10 +78,12 @@ export type ConnectorErrorCode = export class ConnectorTaskError extends Error { readonly code: ConnectorErrorCode + readonly status?: number - constructor(code: ConnectorErrorCode, message: string) { + constructor(code: ConnectorErrorCode, message: string, status?: number) { super(message) this.code = code + this.status = status this.name = 'ConnectorTaskError' } diff --git a/apps/server/node/deployment/connector-transport.ts b/apps/server/node/deployment/connector-transport.ts index 144f214e..4306582e 100644 --- a/apps/server/node/deployment/connector-transport.ts +++ b/apps/server/node/deployment/connector-transport.ts @@ -18,13 +18,11 @@ interface GetOptions { export class RequestError extends ConnectorTaskError { readonly reason: string - readonly status: number | undefined constructor(cause: ConnectorTaskError, reason: string, status: number | undefined) { - super(cause.code, cause.message) + super(cause.code, cause.message, status) this.cause = cause this.reason = reason - this.status = status } } diff --git a/apps/server/node/deployment/connector.ts b/apps/server/node/deployment/connector.ts index 319c88b5..81adea13 100644 --- a/apps/server/node/deployment/connector.ts +++ b/apps/server/node/deployment/connector.ts @@ -454,18 +454,20 @@ export class ConnectorClient implements ConnectorHost { (access.scope != 'trigger' || access.triggerId != triggerId || access.providerId != provider || access.connectionId != connectionId) ) throw accessInvalid() - const bindings = await this.#providerAccessBindings(access, provider, signal, true) + const sourceIngress = request.operation === 'sourceReference' && (request.request.action === 'delete' || request.request.action === 'route') + if (sourceIngress && (typeof access !== 'object' || access.scope !== 'catalog' || access.source !== 'operator')) throw accessInvalid() + const bindings = sourceIngress ? null : await this.#providerAccessBindings(access, provider, signal, true) if (bindings != null && !bindings.some((candidate) => candidate.appId == connectionId && providerAccessAllowsTrigger(candidate, triggerId))) throw accessInvalid() const teamId = connectorTeamId(access) - await this.#assertConnection(connectionId, provider, signal, teamId) + if (!sourceIngress) await this.#assertConnection(connectionId, provider, signal, teamId) const response = await this.#transport.request( 'trigger.execute', - `v1/providers/${encodeURIComponent(provider)}/triggers/${encodeURIComponent(triggerId)}/execute`, + `v1/providers/${encodeURIComponent(provider)}/triggers/${encodeURIComponent(triggerId)}/execute?appId=${encodeURIComponent(connectionId)}`, { method: 'POST', body: JSON.stringify(request), - headers: { 'content-type': 'application/json', 'x-oo-connector-app-id': connectionId }, + headers: { 'content-type': 'application/json' }, }, signal, { fields: { connectionId, provider, triggerId }, teamId }, @@ -485,6 +487,8 @@ export class ConnectorClient implements ConnectorHost { 'connector.input-invalid', 'Webhook creation has an uncertain outcome. Check the existing remote webhook before retrying; do not create another event source.', ) + if (request.operation === 'sourceReference' && response.status === 409 && record(response.value) && response.value.errorCode === 'invalid_input') + throw new ConnectorTaskError('connector.input-invalid', 'The Source changed. Refresh its configuration before retrying.', 409) if (record(response.value) && ['invalid_input', 'trigger_not_found'].includes(String(response.value.errorCode)) && response.status != 409) throw new ConnectorTaskError('connector.input-invalid', 'The Trigger operation configuration is invalid.') if ( diff --git a/apps/server/node/runtime/event-source-runtime.ts b/apps/server/node/runtime/event-source-runtime.ts index da80a90f..6bc2910f 100644 --- a/apps/server/node/runtime/event-source-runtime.ts +++ b/apps/server/node/runtime/event-source-runtime.ts @@ -1,5 +1,5 @@ import type { CreateEventSource, EventSource, UpdateEventSource } from '@oomol-lab/open-flow/control-api' -import type { SourceSubscription as ResourceSubscription, SourceConfiguration, SourceOperation } from '@oomol-lab/open-flow/event-source' +import type { SourceReferenceOperation } from '@oomol-lab/open-flow/event-source' import type { IntegrationDefinition, IntegrationReconcileContext, IntegrationReconcileResult } from '@oomol-lab/open-flow/integration-trigger' import type { ConnectorAccessHost } from '../deployment/connector-access.ts' import type { ConnectorAccessContext, ConnectorHost } from '../deployment/connector-contract.ts' @@ -8,9 +8,19 @@ import type { Store } from '../storage/store.ts' import type { IntegrationOptions } from './integration-runtime.ts' import { controlErrorCode } from '@oomol-lab/open-flow/control-api' -import { SourceIdentityError, eventSourceDefinition, sourceResultSchema, preparedSourceSchema, sourceReceiptSchema } from '@oomol-lab/open-flow/event-source' +import { + SourceIdentityError, + eventSourceDefinition, + sourceReferenceResultSchema, + sourceReferenceDeleteResultSchema, + sourceReferenceResourceResultSchema, + sourceReferenceRouteResultSchema, + sourceReferenceReceiptSchema, + preparedSourceReferenceSchema, +} from '@oomol-lab/open-flow/event-source' import { PermanentIntegrationError, TransientIntegrationError } from '@oomol-lab/open-flow/integration-trigger' -import { z } from 'zod' +import { createHash } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' import { ConnectorTaskError } from '../deployment/connector-contract.ts' import { ControlError } from '../error.ts' @@ -41,7 +51,7 @@ export class EventSourceRuntime { async list(teamId?: string | null, signal?: AbortSignal): Promise<{ readonly version: 1; readonly sources: readonly EventSource[] }> { signal?.throwIfAborted() - const pending = this.#store.eventSources.unprojected(teamId)[Symbol.iterator]() + const pending = this.#store.eventSources.ids(teamId)[Symbol.iterator]() const budget = AbortSignal.timeout(3_000) const inspectionSignal = signal == null ? budget : AbortSignal.any([signal, budget]) const workers = Array.from({ length: 4 }, async () => { @@ -80,32 +90,112 @@ export class EventSourceRuntime { if (connection == null) { throw new ControlError(controlErrorCode.eventSourceInvalid, 'Choose an active Connection in this Connector Team.') } - const { source: configuration } = sourceResultSchema.parse( - await this.#execute(definition.kind, input.connectionId, { operation: 'source', request: { action: 'initialize', input: { ...input } } }, signal, access), + const fingerprint = createHash('sha256').update(JSON.stringify(input)).digest('hex') + const source = this.#store.eventSources.reserve(input, fingerprint) + if (source.pending == null) return this.#store.eventSources.view(await this.#hydrate(source, signal), this.#options()?.publicOrigin) + const result = sourceReferenceResultSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'create', + deploymentId: source.connector!.deploymentId, + requestKey: source.connector!.requestKey, + input: Object.fromEntries( + Object.entries(input).filter(([key]) => !['kind', 'version', 'name', 'connectionId', 'teamId', 'enabled', 'expectedRevision'].includes(key)), + ), + }, + }, + signal, + access, + ), ) - signal.throwIfAborted() - const source = this.#store.eventSources.create(input, definition.kind, configuration) - return this.#store.eventSources.view(source, this.#options()?.publicOrigin) + if (!this.#store.eventSources.saveReference(source, result.source, 'create')) + throw new ControlError(controlErrorCode.eventSourceConflict, 'Source creation is unfinished. Retry the same request.') + return this.#store.eventSources.view(this.#store.eventSources.require(source.sourceId), this.#options()?.publicOrigin) } async update(sourceId: string, input: UpdateEventSource, signal: AbortSignal): Promise { const original = this.#store.eventSources.require(sourceId, input.expectedRevision) - const { source: configuration } = sourceResultSchema.parse( - await this.#execute( - original.kind, - original.connectionId, - { operation: 'source', request: { action: 'update', source: original, input: { ...input } } }, - signal, - this.#operatorContext(original.provider, original.teamId ?? undefined), - ), + if (original.connector?.reference == null) throw new ControlError(controlErrorCode.eventSourceConflict, 'Complete Source creation before editing.') + if ( + original.pending == null && + input.signingSecret == null && + input.encryptKey == null && + input.verificationToken == null && + isDeepStrictEqual(input.eventTypes, original.eventTypes) + ) { + if (this.#store.eventSources.busy(original)) throw new ControlError(controlErrorCode.eventSourceConflict, 'An application operation is unfinished.') + const updated = this.#store.eventSources.update(sourceId, input) + this.#wake() + return this.#store.eventSources.view(updated, this.#options()?.publicOrigin) + } + const inspected = original.pending == null ? await this.#hydrate(original, signal) : original + const source = this.#store.eventSources.beginChange( + sourceId, + input.expectedRevision, + { + action: 'update', + fingerprint: createHash('sha256').update(JSON.stringify(input)).digest('hex'), + remoteRevision: inspected.connector!.revision!, + }, + input, ) - signal.throwIfAborted() - const source = this.#store.eventSources.update(sourceId, input, configuration) + let result + try { + result = sourceReferenceResultSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'update', + deploymentId: source.connector!.deploymentId, + reference: source.connector!.reference!, + expectedRevision: source.pending!.remoteRevision!, + input: Object.fromEntries( + Object.entries(input).filter(([key]) => !['kind', 'version', 'name', 'connectionId', 'teamId', 'enabled', 'expectedRevision'].includes(key)), + ), + }, + }, + signal, + this.#operatorContext(source.provider, source.teamId ?? undefined), + ), + ) + } catch (error) { + if (error instanceof ControlError || (error instanceof ConnectorTaskError && error.code === 'connector.input-invalid')) + this.#store.eventSources.failChange(source) + throw error + } + if (!this.#store.eventSources.saveReference(source, result.source, input)) + throw new ControlError(controlErrorCode.eventSourceConflict, 'Source update is unfinished. Retry the same request.') this.#wake() - return this.#store.eventSources.view(source, this.#options()?.publicOrigin) + return this.#store.eventSources.view(this.#store.eventSources.require(sourceId), this.#options()?.publicOrigin) } - delete(sourceId: string, revision: number): void { + async delete(sourceId: string, revision: number, signal: AbortSignal): Promise { + { + const source = this.#store.eventSources.beginChange(sourceId, revision, { action: 'delete', fingerprint: String(revision) }) + sourceReferenceDeleteResultSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'delete', + deploymentId: source.connector!.deploymentId, + reference: source.connector!.reference ?? { appId: String(source.config.appId), requestKey: source.connector!.requestKey }, + }, + }, + signal, + this.#operatorContext(source.provider, source.teamId ?? undefined), + ), + ) + } this.#store.eventSources.delete(sourceId, revision) } @@ -118,7 +208,12 @@ export class EventSourceRuntime { } const sources = this.#store.eventSources.applicationSources(kind, appId).filter((source) => source.enabled === 1) if (sources.length === 0) return new Response(null, { status: 404 }) - return this.#receive(sources, request) + try { + return await this.#receive(sources, request, true) + } catch { + request.signal.throwIfAborted() + return new Response(null, { status: 503 }) + } } async receive(sourceId: string, request: Request): Promise { @@ -128,7 +223,7 @@ export class EventSourceRuntime { return this.#receive([source], request) } - async #receive(sources: readonly StoredEventSource[], request: Request): Promise { + async #receive(sources: readonly StoredEventSource[], request: Request, application = false): Promise { if (request.method != 'POST') return new Response(null, { status: 405, headers: { allow: 'POST' } }) const definition = eventSourceDefinition(sources[0]!.kind) if (request.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() != 'application/json') return new Response(null, { status: 415 }) @@ -156,6 +251,52 @@ export class EventSourceRuntime { bytes.set(chunk, offset) offset += chunk.length } + if (application) { + const groups = Map.groupBy(sources, (source) => + JSON.stringify([source.teamId, source.connector!.deploymentId, source.connector!.reference?.applicationId]), + ) + const selected: StoredEventSource[] = [] + let response: Response | undefined + for (const group of groups.values()) { + const source = group[0]! + if (this.#store.eventSources.busy(source) || !source.connector?.reference) return new Response(null, { status: 503 }) + const routed = sourceReferenceRouteResultSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'route', + deploymentId: source.connector.deploymentId, + applicationId: source.connector.reference.applicationId, + headers: Object.fromEntries(request.headers), + rawBody: Buffer.from(bytes).toString('base64'), + }, + }, + request.signal, + this.#operatorContext(source.provider, source.teamId ?? undefined), + ), + ) + request.signal.throwIfAborted() + if (routed.response) { + const result = new Response([204, 205, 304].includes(routed.response.status) ? null : routed.response.body, { + status: routed.response.status, + headers: routed.response.headers, + }) + if (!result.ok) return result + response ??= result + } + const ids = new Set( + routed.references + .filter((reference) => reference.applicationId === source.connector!.reference!.applicationId) + .map((reference) => reference.sourceId), + ) + selected.push(...group.filter((item) => ids.has(item.connector!.reference!.sourceId))) + } + if (!selected.length) return response ?? Response.json({}) + sources = selected + } let response: Response | undefined for (const source of sources) { const result = await this.#deliver(source.sourceId, bytes, request.headers, request.signal) @@ -170,30 +311,33 @@ export class EventSourceRuntime { if (source == null || source.enabled != 1) return new Response(null, { status: 404 }) let received try { - received = sourceReceiptSchema.parse( + if (this.#store.eventSources.busy(source)) return new Response(null, { status: 503 }) + const callback = { + action: 'receive' as const, + headers: Object.fromEntries(headers), + rawBody: Buffer.from(bytes).toString('base64'), + consumers: this.#store.eventSources.consumersForEvent(source), + } + const access = this.#operatorContext(source.provider, source.teamId ?? undefined) + const result = sourceReferenceReceiptSchema.parse( await this.#execute( source.kind, source.connectionId, { - operation: 'source', - request: { - action: 'receive', - source, - headers: Object.fromEntries(headers), - rawBody: Buffer.from(bytes).toString('base64'), - consumers: this.#store.eventSources.consumersForEvent(source), - }, + operation: 'sourceReference', + request: { ...callback, deploymentId: source.connector!.deploymentId, reference: source.connector!.reference! }, }, signal, - this.#operatorContext(source.provider, source.teamId ?? undefined), + access, ), ) + if (result.source != null && !this.#store.eventSources.saveReference(source, result.source)) return new Response(null, { status: 409 }) + received = { ...result, source: undefined } } catch { signal.throwIfAborted() return new Response(null, { status: 503 }) } signal.throwIfAborted() - if (received.source != null && !this.#store.eventSources.saveState(source, received.source.state)) return new Response(null, { status: 409 }) if (received.response != null) return new Response([204, 205, 304].includes(received.response.status) ? null : received.response.body, { status: received.response.status, @@ -228,41 +372,32 @@ export class EventSourceRuntime { ) { throw new PermanentIntegrationError('The event source does not authorize this Connection and Flow scope.') } + if (this.#store.eventSources.busy(source)) throw new TransientIntegrationError('A Source operation is unfinished.') source = await this.#hydrate(source) const sourceDefinition = eventSourceDefinition(source.kind) if (source.enabled != 1) throw new PermanentIntegrationError('Enable the event source before publishing.') - let subscriptions: readonly ResourceSubscription[] - try { - const prepared = preparedSourceSchema.parse( - await this.#execute( - source.kind, - source.connectionId, - { - operation: 'source', - request: { - action: 'prepare', - source, - config: context.config, - endpointUrl: this.#store.eventSources.view(source, this.#options()?.publicOrigin).endpointUrl ?? undefined, - }, + const prepared = preparedSourceReferenceSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'prepare', + deploymentId: source.connector!.deploymentId, + reference: source.connector!.reference!, + config: context.config, + endpointUrl: this.#store.eventSources.view(source, this.#options()?.publicOrigin).endpointUrl ?? undefined, }, - context.signal ?? AbortSignal.timeout(30_000), - access, - ), - ) - if (!this.#store.eventSources.saveState(source, prepared.source.state)) throw new TransientIntegrationError('The source changed during preparation.') - subscriptions = prepared.subscriptions - } catch (error) { - if (sourceDefinition.managed) - this.#store.eventSources.saveState( - source, - this.#resourceState(source, 'error', 'The source could not be verified. Check authorization and retry publication.'), - ) - throw error - } + }, + context.signal ?? AbortSignal.timeout(30_000), + access, + ), + ) + if (!this.#store.eventSources.saveReference(source, prepared.source)) throw new TransientIntegrationError('The Source changed during preparation.') + const subscriptions = prepared.subscriptions for (const subscription of subscriptions) { if (access.scope == 'catalog') throw new PermanentIntegrationError('Resource subscriptions require a fixed access snapshot.') - const previous = this.#store.eventSources.subscription(source.sourceId, subscription.key) const stored = this.#store.eventSources.demand( source.sourceId, subscription.key, @@ -278,8 +413,6 @@ export class EventSourceRuntime { await running return { outcome: 'pending' } } - if (previous != null && !sourceDefinition.managed) - throw new PermanentIntegrationError('A previous subscription operation has an uncertain outcome. Verify it before retrying.') const operation = this.#change(source, stored, true, context.signal) this.#operations.set(key, operation) try { @@ -298,7 +431,7 @@ export class EventSourceRuntime { const key = JSON.stringify([subscription.sourceId, subscription.resourceKey]) if (this.#operations.has(key)) continue const source = this.#store.eventSources.get(subscription.sourceId) - if (source == null || (subscription.status != 'ready' && !eventSourceDefinition(source.kind).managed)) continue + if (source == null) continue this.#store.eventSources.subscriptionState(source.sourceId, subscription.resourceKey, 'deleting', this.#clock()) const operation = this.#change(source, subscription, false, signal) this.#operations.set(key, operation) @@ -314,101 +447,68 @@ export class EventSourceRuntime { } async #change(source: StoredEventSource, stored: SourceSubscription, active: boolean, signal?: AbortSignal): Promise { - const subscription = JSON.parse(stored.resourceJson) as ResourceSubscription - const { triggerId, managed } = eventSourceDefinition(source.kind) - const connector = this.#connector() - if (connector == null) throw new TransientIntegrationError('Connector is unavailable.') + const { triggerId } = eventSourceDefinition(source.kind) try { - if (connector.trigger == null) throw new TransientIntegrationError('Trigger operation transport is unavailable.') - if (managed && !this.#store.eventSources.saveState(source, this.#resourceState(source, 'pending', ''))) - throw new TransientIntegrationError('The source changed during subscription setup.') - const result = await connector.trigger( - source.provider, - source.connectionId, - triggerId, - { operation: 'resource', config: subscription.config, requestKey: `${source.sourceId}:${stored.resourceKey}`, active }, - signal ?? AbortSignal.timeout(30_000), - { - providerAccess: stored.providerAccess, - providerId: source.provider, - scope: 'trigger', - triggerId, - connectionId: source.connectionId, - purpose: 'trigger', - source: 'publication', - ...(source.teamId == null ? {} : { teamId: source.teamId }), - }, - ) - if (managed) { - const confirmed = z.object({ outcome: z.literal('ready'), subscription: z.record(z.string(), z.json()) }).safeParse(result) - if (!confirmed.success) throw new TransientIntegrationError('Connector did not confirm the source subscription.') - const next = { - ...source, - state: { - ...source.state, - resource: confirmed.data.subscription, - resourceStatus: active ? 'ready' : 'idle', - lastError: '', - verifiedAt: active ? this.#clock() : null, + const result = sourceReferenceResourceResultSchema.parse( + await this.#execute( + source.kind, + source.connectionId, + { + operation: 'sourceReference', + request: { + action: 'resource', + deploymentId: source.connector!.deploymentId, + reference: source.connector!.reference!, + key: stored.resourceKey, + active, + }, }, - } - const inspected = sourceResultSchema.parse( - await this.#execute( - source.kind, - source.connectionId, - { operation: 'source', request: { action: 'inspect', source: next } }, - signal ?? AbortSignal.timeout(30_000), - this.#operatorContext(source.provider, source.teamId ?? undefined), - ), - ) - if (!this.#store.eventSources.saveState(source, inspected.source.state)) - throw new TransientIntegrationError('The source changed during subscription setup.') - } + signal ?? AbortSignal.timeout(30_000), + { + providerAccess: stored.providerAccess, + providerId: source.provider, + scope: 'trigger', + triggerId, + connectionId: source.connectionId, + purpose: 'trigger', + source: 'publication', + ...(source.teamId == null ? {} : { teamId: source.teamId }), + }, + ), + ) + if (!this.#store.eventSources.saveReference(source, result.source)) throw new TransientIntegrationError('The Source changed during resource preparation.') + if (result.outcome === 'pending') throw new TransientIntegrationError('The Source resource is still being prepared.') if (active) this.#store.eventSources.subscriptionState(source.sourceId, stored.resourceKey, 'ready', this.#clock()) else this.#store.eventSources.deleteSubscription(source.sourceId, stored.resourceKey) } catch (error) { - if (managed) - this.#store.eventSources.saveState( - source, - this.#resourceState(source, 'error', error instanceof ConnectorTaskError ? error.message : 'Source subscription could not be confirmed.'), - ) this.#store.eventSources.subscriptionState(source.sourceId, stored.resourceKey, 'uncertain', this.#clock()) throw error } } async #hydrate(source: StoredEventSource, signal = AbortSignal.timeout(30_000)): Promise { - if (source.state.projection != null) return source - const inspected = sourceResultSchema.parse( + if (source.connector?.reference == null || this.#store.eventSources.busy(source)) return source + const result = sourceReferenceResultSchema.parse( await this.#execute( source.kind, source.connectionId, - { operation: 'source', request: { action: 'inspect', source } }, + { + operation: 'sourceReference', + request: { action: 'get', deploymentId: source.connector.deploymentId, reference: source.connector.reference }, + }, signal, this.#operatorContext(source.provider, source.teamId ?? undefined), ), ) signal.throwIfAborted() - this.#store.eventSources.saveState(source, inspected.source.state) - const current = this.#store.eventSources.get(source.sourceId) - if (current == null) throw new TransientIntegrationError('The source was deleted during protocol initialization.') - return current - } - - #resourceState(source: SourceConfiguration, status: 'pending' | 'error', message: string): SourceConfiguration['state'] { - const projection = source.state.projection as { view: Record } - return { - ...source.state, - resourceStatus: status, - lastError: message, - projection: { ...projection, ready: false, view: { ...projection.view, subscriptionStatus: status, lastError: message } }, - } + if (!this.#store.eventSources.saveReference(source, result.source)) throw new TransientIntegrationError('The Source changed while reading it.') + return this.#store.eventSources.require(source.sourceId) } async #execute( kind: string, connectionId: string, - operation: { operation: 'source'; request: SourceOperation }, + operation: { operation: 'sourceReference'; request: SourceReferenceOperation }, signal: AbortSignal, access: ConnectorAccessContext, ): Promise { @@ -418,8 +518,10 @@ export class EventSourceRuntime { try { return await connector.trigger(definition.provider, connectionId, definition.triggerId, operation, signal, access) } catch (error) { + if (error instanceof ConnectorTaskError && error.code === 'connector.input-invalid' && error.status === 409) + throw new ControlError(controlErrorCode.eventSourceConflict, error.message) if (error instanceof ConnectorTaskError && error.code === 'connector.input-invalid') { - if (operation.request.action === 'initialize') throw new ControlError(controlErrorCode.eventSourceIdentityUnavailable, error.message) + if (operation.request.action === 'create') throw new ControlError(controlErrorCode.eventSourceIdentityUnavailable, error.message) if (operation.request.action === 'update') throw new ControlError(controlErrorCode.eventSourceInvalid, error.message) } throw error diff --git a/apps/server/node/runtime/integration-runtime.ts b/apps/server/node/runtime/integration-runtime.ts index a05ae68f..2f1a0147 100644 --- a/apps/server/node/runtime/integration-runtime.ts +++ b/apps/server/node/runtime/integration-runtime.ts @@ -855,7 +855,7 @@ export class IntegrationRuntime { this.#store.eventSources.finish(delivery, 'retired') continue } - if (source.enabled != 1 || !sourceReady(source) || !this.#store.eventSources.deliverable(delivery)) { + if (source.enabled != 1 || this.#store.eventSources.busy(source) || !sourceReady(source) || !this.#store.eventSources.deliverable(delivery)) { this.#store.eventSources.retry(delivery, now) continue } diff --git a/apps/server/node/storage/event-source-store.ts b/apps/server/node/storage/event-source-store.ts index c49f0519..199b0bdb 100644 --- a/apps/server/node/storage/event-source-store.ts +++ b/apps/server/node/storage/event-source-store.ts @@ -1,21 +1,24 @@ import type { ConnectorAccessSnapshot, CreateEventSource, EventSource, UpdateEventSource } from '@oomol-lab/open-flow/control-api' -import type { SourceConfiguration, SourceEvent } from '@oomol-lab/open-flow/event-source' +import type { SourceConfiguration, SourceEvent, SourceReferenceSnapshot } from '@oomol-lab/open-flow/event-source' import type { TriggerNode } from '@oomol-lab/open-flow/flow-change' import type { DatabaseSync } from 'node:sqlite' import { decodeConnectorAccessSnapshot } from '@oomol-lab/open-flow/control-api' import { controlErrorCode } from '@oomol-lab/open-flow/control-api' -import { SourceIdentityError, eventSourceDefinition, sourceReady, sourceView } from '@oomol-lab/open-flow/event-source' +import { SourceIdentityError, eventSourceDefinition, sourceReady, sourceView, sourceReferenceSchema } from '@oomol-lab/open-flow/event-source' import { resolveTriggerConfig } from '@oomol-lab/open-flow/integration-trigger' import { randomUUID } from 'node:crypto' +import { isDeepStrictEqual } from 'node:util' import { z, ZodError } from 'zod' import { ControlError } from '../error.ts' const sourceColumns = `source_id AS sourceId, revision, name, kind, provider, external_identity AS identity, connection_id AS connectionId, team_id AS teamId, config_json AS configJson, secrets_json AS secretsJson, - state_json AS stateJson, event_types_json AS eventTypesJson, enabled, last_received_at AS lastReceivedAt, updated_at AS updatedAt` + state_json AS stateJson, connector_json AS connectorJson, pending_json AS pendingJson, event_types_json AS eventTypesJson, enabled, last_received_at AS lastReceivedAt, updated_at AS updatedAt` export interface StoredEventSource extends SourceConfiguration { + readonly connector: z.infer | null + readonly pending: z.infer | null readonly sourceId: string readonly revision: number readonly name: string @@ -28,19 +31,31 @@ export interface StoredEventSource extends SourceConfiguration { readonly updatedAt: number } -type SourceRow = Omit & { +type SourceRow = Omit & { + readonly connectorJson: string | null + readonly pendingJson: string | null readonly configJson: string readonly secretsJson: string readonly stateJson: string readonly eventTypesJson: string } +const connectorSchema = z.object({ + deploymentId: z.string(), + requestKey: z.string(), + creation: z.string(), + reference: sourceReferenceSchema.optional(), + revision: z.number().int().positive().optional(), +}) +const pendingSchema = z.object({ action: z.enum(['create', 'update', 'delete']), fingerprint: z.string(), remoteRevision: z.number().optional() }) const storedObject = z.record(z.string(), z.json()) function stored(row: SourceRow): StoredEventSource { - const { configJson, secretsJson, stateJson, eventTypesJson, ...common } = row + const { configJson, secretsJson, stateJson, eventTypesJson, connectorJson, pendingJson, ...common } = row return { ...common, + connector: connectorJson == null ? null : connectorSchema.parse(JSON.parse(connectorJson)), + pending: pendingJson == null ? null : pendingSchema.parse(JSON.parse(pendingJson)), config: storedObject.parse(JSON.parse(configJson)), secrets: storedObject.parse(JSON.parse(secretsJson)), state: storedObject.parse(JSON.parse(stateJson)), @@ -84,29 +99,8 @@ export class EventSourceStore { #source(row: SourceRow): StoredEventSource { const source = stored(row) - if (!eventSourceDefinition(source.kind).applicationIngress) return source - const app = this.#database - .prepare('SELECT signing_secret AS secret, verified_at AS verifiedAt FROM event_source_apps WHERE kind = ? AND app_id = ?') - .get(source.kind, String(source.config.appId)) as { secret: string; verifiedAt: number | null } | undefined - if (app == null) throw new SourceIdentityError('The source application is missing.') - const projection = source.state.projection as { view: Record; ready: boolean } | undefined - return { - ...source, - secrets: { signingSecret: app.secret }, - state: { - ...source.state, - verifiedAt: app.verifiedAt, - ...(projection == null - ? {} - : { - projection: { - ...projection, - ready: app.verifiedAt != null, - view: { ...projection.view, verifiedAt: app.verifiedAt == null ? null : new Date(app.verifiedAt).toISOString() }, - }, - }), - }, - } + if (source.connector == null) throw new SourceIdentityError('Source requires a Connector reference.') + return source } applicationSources(kind: string, appId: string): readonly StoredEventSource[] { @@ -116,26 +110,25 @@ export class EventSourceStore { return rows.map((row) => this.#source(row)) } - unprojected(teamId?: string | null): readonly string[] { + ids(teamId?: string | null): readonly string[] { return this.#database - .prepare( - `SELECT source_id AS sourceId FROM event_sources WHERE json_extract(state_json, '$.projection') IS NULL${teamId === undefined ? '' : ' AND team_id IS ?'}`, - ) + .prepare(`SELECT source_id AS sourceId FROM event_sources${teamId === undefined ? '' : ' WHERE team_id IS ?'}`) .all(...(teamId === undefined ? [] : [teamId])) .map((row) => String(row.sourceId)) } list(origin: string | undefined, teamId?: string | null): readonly EventSource[] { - const rows = this.#database.prepare(`SELECT ${sourceColumns} FROM event_sources ORDER BY name, source_id`).all() as unknown as SourceRow[] + const rows = this.#database + .prepare(`SELECT ${sourceColumns} FROM event_sources${teamId === undefined ? '' : ' WHERE team_id IS ?'} ORDER BY name, source_id`) + .all(...(teamId === undefined ? [] : [teamId])) as unknown as SourceRow[] const result: EventSource[] = [] for (const row of rows) { - if (teamId !== undefined && row.teamId !== teamId) continue try { result.push(this.view(this.#source(row), origin)) } catch (error) { if (!(error instanceof SourceIdentityError || error instanceof ZodError || error instanceof SyntaxError)) throw error result.push({ - ...this.view({ ...row, config: {}, secrets: {}, state: {}, eventTypes: [] }, undefined), + ...this.view({ ...row, connector: null, pending: null, config: {}, secrets: {}, state: {}, eventTypes: [] }, undefined), lastError: 'Stored source details are invalid or unsupported. Repair or recreate this event source.', }) } @@ -155,6 +148,7 @@ export class EventSourceStore { enabled: row.enabled == 1, eventTypes: row.eventTypes, ...sourceView(row, this.#clock()), + ...(row.pending == null ? {} : { lastError: `Source ${row.pending.action} is unfinished. Retry the same operation to complete it.` }), endpointUrl: origin == null ? null @@ -167,121 +161,178 @@ export class EventSourceStore { } } - create(input: CreateEventSource, kind: string, configuration: SourceConfiguration): StoredEventSource { + reserve(input: CreateEventSource, fingerprint: string): StoredEventSource { return this.#transaction(() => { - if (this.#database.prepare('SELECT 1 FROM event_sources WHERE kind = ? AND external_identity = ?').get(kind, configuration.identity) != null) { - throw new ControlError(controlErrorCode.eventSourceConflict, 'This application already has an event source.') - } - const count = this.#database.prepare('SELECT COUNT(*) AS count FROM event_sources').get() as { count: number } - if (count.count >= 100) throw new ControlError(controlErrorCode.eventSourceConflict, 'The deployment event source limit has been reached.') - if (eventSourceDefinition(kind).applicationIngress) { - const appId = String(configuration.config.appId) - const app = this.#database.prepare('SELECT signing_secret AS secret FROM event_source_apps WHERE kind = ? AND app_id = ?').get(kind, appId) as - | { secret: string } - | undefined - if (app != null && app.secret !== configuration.secrets.signingSecret) + const kind = input.kind ?? 'feishu' + const config = Object.fromEntries( + Object.entries(input).filter( + ([key]) => !['signingSecret', 'verificationToken', 'encryptKey', 'eventTypes', 'name', 'version', 'teamId', 'connectionId', 'kind'].includes(key), + ), + ) + const identity = JSON.stringify([input.teamId, input.connectionId, config]) + const existing = this.#database + .prepare('SELECT source_id AS sourceId FROM event_sources WHERE kind = ? AND external_identity = ?') + .get(kind, identity) as { sourceId: string } | undefined + if (existing != null) { + const source = this.require(existing.sourceId) + if (source.connector!.creation !== fingerprint || (source.pending != null && source.pending.action !== 'create')) throw new ControlError( controlErrorCode.eventSourceConflict, - 'This application already has a different Signing Secret. Update the existing application source first.', + 'This workspace already has a Source. Retry the original creation or edit the existing Source.', ) - if (app == null) - this.#database - .prepare('INSERT INTO event_source_apps (kind, app_id, signing_secret) VALUES (?, ?, ?)') - .run(kind, appId, String(configuration.secrets.signingSecret)) + return source } + const count = this.#database.prepare('SELECT COUNT(*) AS count FROM event_sources').get() as { count: number } + if (count.count >= 100) throw new ControlError(controlErrorCode.eventSourceConflict, 'The deployment event source limit has been reached.') + if (eventSourceDefinition(kind).applicationIngress && this.busy({ kind, teamId: input.teamId, config })) + throw new ControlError(controlErrorCode.eventSourceConflict, 'Another Source operation for this application is unfinished.') const sourceId = `source_${randomUUID().replaceAll('-', '')}` + const { value: deploymentId } = this.#database.prepare('SELECT value FROM event_source_namespace WHERE id = 1').get() as { value: string } this.#database .prepare(`INSERT INTO event_sources (source_id, revision, name, kind, provider, external_identity, connection_id, - team_id, config_json, secrets_json, state_json, event_types_json, updated_at) - VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`) + team_id, config_json, secrets_json, state_json, event_types_json, updated_at, connector_json, pending_json) + VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, '{}', '{}', ?, ?, ?, ?)`) .run( sourceId, input.name, kind, eventSourceDefinition(kind).provider, - configuration.identity, + identity, input.connectionId, input.teamId, - JSON.stringify(configuration.config), - JSON.stringify(eventSourceDefinition(kind).applicationIngress ? {} : configuration.secrets), - JSON.stringify(configuration.state), - JSON.stringify(configuration.eventTypes), + JSON.stringify(config), + JSON.stringify(input.eventTypes), this.#clock(), + JSON.stringify({ deploymentId, requestKey: sourceId, creation: fingerprint }), + JSON.stringify({ action: 'create', fingerprint }), ) - return this.get(sourceId)! + return this.require(sourceId) }) } - update(sourceId: string, input: UpdateEventSource, configuration: SourceConfiguration): StoredEventSource { + busy(source: Pick): boolean { + if (!eventSourceDefinition(source.kind).applicationIngress) return 'pending' in source && source.pending != null + return ( + this.#database + .prepare(`SELECT 1 FROM event_sources WHERE kind = ? AND team_id IS ? + AND json_extract(config_json, '$.appId') = ? AND pending_json IS NOT NULL LIMIT 1`) + .get(source.kind, source.teamId, String(source.config.appId)) != null + ) + } + + beginChange(sourceId: string, revision: number, pending: z.infer, input?: UpdateEventSource): StoredEventSource { return this.#transaction(() => { - const source = this.require(sourceId, input.expectedRevision) - const required = this.#database - .prepare(`SELECT trigger_json AS triggerJson FROM integration_bindings - WHERE current_publication_id IS NOT NULL AND json_extract(trigger_json, '$.config.sourceId.value') = ? - UNION ALL SELECT trigger_json AS triggerJson FROM integration_candidates - WHERE status != 'cleanup' AND json_extract(trigger_json, '$.config.sourceId.value') = ?`) - .all(sourceId, sourceId) as { triggerJson: string }[] - for (const row of required) { - const trigger = JSON.parse(row.triggerJson) as Extract - const config = resolveTriggerConfig(trigger.definition.configInputs, trigger.config) - const eventTypes = config.eventTypes as readonly string[] - if (eventTypes.some((type) => !configuration.eventTypes.includes(type))) { - throw new ControlError(controlErrorCode.eventSourceConflict, 'Published or preparing Triggers still require these event types.') - } - } - if (eventSourceDefinition(source.kind).applicationIngress && input.signingSecret != null && input.signingSecret !== source.secrets.signingSecret) { - this.#database - .prepare('UPDATE event_source_apps SET signing_secret = ?, verified_at = NULL WHERE kind = ? AND app_id = ?') - .run(input.signingSecret, source.kind, String(source.config.appId)) - this.#database - .prepare( - "UPDATE event_sources SET revision = revision + 1, updated_at = ? WHERE kind = ? AND json_extract(config_json, '$.appId') = ? AND source_id != ?", - ) - .run(this.#clock(), source.kind, String(source.config.appId), sourceId) + const source = this.require(sourceId, revision) + if (source.pending != null && pending.action !== 'delete') { + if (source.pending.action !== pending.action || source.pending.fingerprint !== pending.fingerprint) + throw new ControlError(controlErrorCode.eventSourceConflict, 'Retry the unfinished Source operation before making another change.') + return source } + if (source.pending == null && this.busy(source)) + throw new ControlError(controlErrorCode.eventSourceConflict, 'Another Source operation for this application is unfinished.') + if (pending.action === 'delete') this.#checkDelete(sourceId) + if (input != null) this.#checkEvents(sourceId, input.eventTypes) + this.#database.prepare('UPDATE event_sources SET pending_json = ? WHERE source_id = ?').run(JSON.stringify(pending), sourceId) + return this.require(sourceId) + }) + } + + saveReference(source: StoredEventSource, snapshot: SourceReferenceSnapshot, change?: UpdateEventSource | 'create'): boolean { + const input = typeof change === 'object' ? change : undefined + const complete = change != null + return this.#transaction(() => { + const current = this.require(source.sourceId) + if (current.revision !== source.revision || (complete && !isDeepStrictEqual(current.pending, source.pending))) return false + if (!complete && this.busy(current)) return false + if ( + current.connector!.reference != null && + (current.connector!.reference.applicationId !== snapshot.reference.applicationId || + current.connector!.reference.sourceId !== snapshot.reference.sourceId) + ) + throw new SourceIdentityError('Connector returned a different Source reference.') + if ((current.connector!.revision ?? 0) > snapshot.revision) return false + if (complete && input != null) this.#checkEvents(source.sourceId, snapshot.eventTypes) this.#database - .prepare(`UPDATE event_sources SET name = ?, enabled = ?, event_types_json = ?, config_json = ?, secrets_json = ?, - state_json = ?, revision = revision + 1, updated_at = ? WHERE source_id = ?`) + .prepare(`UPDATE event_sources SET connector_json = ?, state_json = ?, event_types_json = ?, + name = ?, enabled = ?, pending_json = ?, revision = revision + ?, updated_at = ? WHERE source_id = ?`) .run( - input.name, - Number(input.enabled), - JSON.stringify(configuration.eventTypes), - JSON.stringify(configuration.config), - JSON.stringify(eventSourceDefinition(source.kind).applicationIngress ? {} : configuration.secrets), - JSON.stringify(configuration.state), - this.#clock(), - sourceId, + JSON.stringify({ ...current.connector, reference: snapshot.reference, revision: snapshot.revision }), + JSON.stringify({ projection: { view: snapshot.view, ready: snapshot.ready, expiresAt: snapshot.expiresAt } }), + JSON.stringify(snapshot.eventTypes), + input?.name ?? current.name, + input == null ? current.enabled : Number(input.enabled), + complete || current.pending == null ? null : JSON.stringify(current.pending), + input == null ? 0 : 1, + complete ? this.#clock() : current.updatedAt, + source.sourceId, ) - return this.get(sourceId)! + if (complete && input?.signingSecret != null && eventSourceDefinition(current.kind).applicationIngress) { + this.#database + .prepare(`UPDATE event_sources SET state_json = '{}', revision = revision + 1 WHERE kind = ? + AND team_id IS ? AND json_extract(config_json, '$.appId') = ? AND source_id != ?`) + .run(current.kind, current.teamId, String(current.config.appId), current.sourceId) + } + return true }) } + failChange(source: StoredEventSource): void { + this.#database + .prepare('UPDATE event_sources SET pending_json = NULL WHERE source_id = ? AND revision = ? AND pending_json = ?') + .run(source.sourceId, source.revision, JSON.stringify(source.pending)) + } + + update(sourceId: string, input: UpdateEventSource): StoredEventSource { + return this.#transaction(() => { + this.require(sourceId, input.expectedRevision) + this.#database + .prepare('UPDATE event_sources SET name = ?, enabled = ?, revision = revision + 1, updated_at = ? WHERE source_id = ?') + .run(input.name, Number(input.enabled), this.#clock(), sourceId) + return this.require(sourceId) + }) + } + + #checkEvents(sourceId: string, eventTypes: readonly string[]): void { + const required = this.#database + .prepare(`SELECT trigger_json AS triggerJson FROM integration_bindings + WHERE current_publication_id IS NOT NULL AND json_extract(trigger_json, '$.config.sourceId.value') = ? + UNION ALL SELECT trigger_json AS triggerJson FROM integration_candidates + WHERE status != 'cleanup' AND json_extract(trigger_json, '$.config.sourceId.value') = ?`) + .all(sourceId, sourceId) as { triggerJson: string }[] + for (const row of required) { + const trigger = JSON.parse(row.triggerJson) as Extract + const config = resolveTriggerConfig(trigger.definition.configInputs, trigger.config) + const requiredTypes = config.eventTypes as readonly string[] + if (requiredTypes.some((type) => !eventTypes.includes(type))) { + throw new ControlError(controlErrorCode.eventSourceConflict, 'Published or preparing Triggers still require these event types.') + } + } + } + delete(sourceId: string, revision: number): void { this.#transaction(() => { - const source = this.require(sourceId, revision) - if ( - this.consumers(sourceId).length > 0 || - this.#database.prepare('SELECT 1 FROM source_subscriptions WHERE source_id = ?').get(sourceId) != null || - this.#database - .prepare("SELECT 1 FROM integration_candidates WHERE status != 'cleanup' AND json_extract(trigger_json, '$.config.sourceId.value') = ?") - .get(sourceId) != null - ) { - throw new ControlError( - controlErrorCode.eventSourceConflict, - 'Remove the source from published Triggers and finish subscription cleanup before deleting it.', - ) - } + this.require(sourceId, revision) + this.#checkDelete(sourceId) for (const table of ['source_deliveries', 'source_events', 'source_event_versions', 'source_demands', 'event_sources']) this.#database.prepare(`DELETE FROM ${table} WHERE source_id = ?`).run(sourceId) - if (eventSourceDefinition(source.kind).applicationIngress) - this.#database - .prepare( - "DELETE FROM event_source_apps WHERE kind = ? AND app_id = ? AND NOT EXISTS (SELECT 1 FROM event_sources WHERE kind = ? AND json_extract(config_json, '$.appId') = ?)", - ) - .run(source.kind, String(source.config.appId), source.kind, String(source.config.appId)) }) } + #checkDelete(sourceId: string): void { + if ( + this.consumers(sourceId).length > 0 || + this.#database.prepare('SELECT 1 FROM source_subscriptions WHERE source_id = ?').get(sourceId) != null || + this.#database + .prepare("SELECT 1 FROM integration_candidates WHERE status != 'cleanup' AND json_extract(trigger_json, '$.config.sourceId.value') = ?") + .get(sourceId) != null + ) { + throw new ControlError( + controlErrorCode.eventSourceConflict, + 'Remove the source from published Triggers and finish subscription cleanup before deleting it.', + ) + } + } + require(sourceId: string, revision?: number): StoredEventSource { const row = this.get(sourceId) if (row == null) throw new ControlError(controlErrorCode.eventSourceNotFound, 'Event source was not found.') @@ -299,23 +350,6 @@ export class EventSourceStore { .all(sourceId) as unknown as EventSource['consumers'] } - saveState(source: StoredEventSource, state: SourceConfiguration['state']): boolean { - return this.#transaction(() => { - if (eventSourceDefinition(source.kind).applicationIngress) { - const current = this.get(source.sourceId) - if (current == null || current.revision !== source.revision) return false - this.#database - .prepare('UPDATE event_source_apps SET verified_at = ? WHERE kind = ? AND app_id = ?') - .run(typeof state.verifiedAt === 'number' ? state.verifiedAt : null, source.kind, String(source.config.appId)) - } - return ( - this.#database - .prepare('UPDATE event_sources SET state_json = ? WHERE source_id = ? AND revision = ?') - .run(JSON.stringify(state), source.sourceId, source.revision).changes == 1 - ) - }) - } - private targets(source: StoredEventSource) { return this.#database .prepare(`SELECT b.binding_id AS bindingId, b.endpoint_id AS endpointId, b.current_publication_id AS publicationId, @@ -343,7 +377,8 @@ export class EventSourceStore { receive(source: StoredEventSource, event: SourceEvent, now: number): 'accepted' | 'conflict' | 'overloaded' | 'unavailable' { return this.#transaction(() => { const current = this.get(source.sourceId) - if (current == null || current.revision != source.revision || current.enabled != 1 || !sourceReady(current, now)) return 'unavailable' + if (current == null || current.revision != source.revision || current.enabled != 1 || this.busy(current) || !sourceReady(current, now)) + return 'unavailable' const payloadJson = JSON.stringify(event.payload) const existing = this.#database .prepare('SELECT payload_json AS payloadJson FROM source_events WHERE source_id = ? AND event_id = ?') @@ -479,6 +514,7 @@ export class EventSourceStore { const source = this.get(sourceId) return ( source?.enabled == 1 && + !this.busy(source) && sourceReady(source, this.#clock()) && this.#database .prepare(`SELECT 1 FROM source_demands d diff --git a/apps/server/node/storage/integration-store.ts b/apps/server/node/storage/integration-store.ts index 4fb396fb..41d00d50 100644 --- a/apps/server/node/storage/integration-store.ts +++ b/apps/server/node/storage/integration-store.ts @@ -80,6 +80,11 @@ export class IntegrationStore { ): boolean { const candidates = [] for (const integration of integrations) { + if (integration.eventSource) { + const sourceId = (JSON.parse(integration.triggerJson) as { config?: { sourceId?: { value?: string } } }).config?.sourceId?.value + const source = sourceId == null ? undefined : this.#eventSources.get(sourceId) + if (source == null || this.#eventSources.busy(source)) return false + } const current = this.integrationBinding(flowId, integration.triggerNodeId) if (current != null) { if (this.#canReuse(current, integration, expectedLivePublicationId)) continue @@ -393,7 +398,9 @@ export class IntegrationStore { integration: Pick, expectedLivePublicationId: string | null, ): boolean { + const sourceId = (JSON.parse(integration.triggerJson) as { config?: { sourceId?: { value?: string } } }).config?.sourceId?.value return ( + (sourceId == null || this.#eventSources.ready(sourceId, binding.bindingId)) && binding.currentPublicationId == expectedLivePublicationId && this.#matches(binding, integration) && (binding.health == 'healthy' || (integration.listener === true && this.listenerState(binding.bindingId, binding.runtimeVersion)?.health == 'healthy')) diff --git a/apps/server/node/storage/migrate.ts b/apps/server/node/storage/migrate.ts index b1a88fa0..cf0de8be 100644 --- a/apps/server/node/storage/migrate.ts +++ b/apps/server/node/storage/migrate.ts @@ -60,6 +60,7 @@ const migrations = [ '0051_event_source_apps.sql', '0052_webhook_responses.sql', '0053_wait_auto_continue.sql', + '0054_source_references.sql', ] as const const migrationsDirectory = new URL(import.meta.url.endsWith('.ts') ? '../../migrations/' : '../migrations/', import.meta.url) diff --git a/apps/server/node/transport/control.ts b/apps/server/node/transport/control.ts index 8f8e8243..2ca68912 100644 --- a/apps/server/node/transport/control.ts +++ b/apps/server/node/transport/control.ts @@ -149,7 +149,7 @@ export function createControlApp(service: ControlService, resolveActor?: Resolve app.delete('/event-sources/:sourceId', async (context) => { query(context.req.raw, [], controlErrorCode.eventSourceInvalid) const body = await decodeRequest(context.req.raw, controlErrorCode.eventSourceInvalid, controlRequests.eventSourceRevision) - service.eventSources.deleteEventSource(context.get('actorId'), context.req.param('sourceId'), body.expectedRevision) + await service.eventSources.deleteEventSource(context.get('actorId'), context.req.param('sourceId'), body.expectedRevision, context.req.raw.signal) return response(200, { version: 1 }) }) diff --git a/apps/server/scripts/check-trigger-source.ts b/apps/server/scripts/check-trigger-source.ts new file mode 100644 index 00000000..25b76db1 --- /dev/null +++ b/apps/server/scripts/check-trigger-source.ts @@ -0,0 +1,196 @@ +import type { ConnectorHost } from '../node/deployment/connector-contract.ts' + +import { decodeEventSource } from '@oomol-lab/open-flow/control-api' +import { inputValues } from '@oomol-lab/open-flow/flow-change' +import { integrationDefinitions } from '@oomol-lab/open-flow/provider-triggers' +import { currentEngineContract } from '@oomol-lab/open-flow/runtime-contract' +import * as Effect from 'effect/Effect' +import * as Exit from 'effect/Exit' +import * as Scope from 'effect/Scope' +import assert from 'node:assert/strict' +import { createHmac } from 'node:crypto' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { ServerService } from '../node/application/service.ts' +import { Database } from '../node/storage/database.ts' +import { createServerApp } from '../node/transport/http.ts' + +const adapterPath = process.argv[2] +if (!adapterPath) throw new Error('Pass the Connector conformance adapter path.') +const adapter = (await import(pathToFileURL(resolve(adapterPath)).href)) as { + createHarness(upstream: () => Promise): Promise<{ + request(triggerId: string, body: unknown): Promise + disconnect(): Promise + close(): Promise + }> +} +const unexpected = async (): Promise => { + throw new Error('Unexpected upstream operation') +} +const harness = await adapter.createHarness(unexpected) +const database = Database.open(':memory:') +const scope = await Effect.runPromise(Scope.make()) +try { + let callbackCalls = 0 + const connector: ConnectorHost = { + execute: unexpected, + getAction: unexpected, + listActions: unexpected, + listAllConnections: unexpected, + listProviders: unexpected, + proxy: unexpected, + searchActions: unexpected, + ready: async () => true, + listConnections: async () => [ + { connectionId: 'connection', providerAccountId: 'U123', serviceId: 'slackbot', displayName: 'Slack', status: 'active', isDefault: true }, + { connectionId: 'unrelated', providerAccountId: 'U123', serviceId: 'slackbot', displayName: 'Unrelated', status: 'active', isDefault: false }, + ], + trigger: async (_provider, _connection, triggerId, request) => { + assert.equal(request.operation, 'sourceReference') + if (_connection === 'unrelated' && request.request.action === 'receive') throw new Error('Unrelated connection unavailable') + if (request.request.action === 'route' || request.request.action === 'receive') callbackCalls++ + const response = await harness.request(triggerId, request) + const body = (await response.json()) as { data?: unknown } + assert.equal(response.status, 200, JSON.stringify(body)) + return body.data + }, + } + const service = await Effect.runPromise( + ServerService.open(database, { + capabilities: { connector: () => connector, integration: () => ({ callbackKey: 'key', publicOrigin: 'https://flow.example' }) }, + }).pipe(Scope.provide(scope)), + ) + const app = createServerApp(service, { resolveControlActor: () => 'operator' }) + const create = await app.request('/v1/event-sources', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + kind: 'slack', + version: 1, + name: 'Slack', + connectionId: 'connection', + teamId: null, + appId: 'A123', + workspaceId: 'T123', + signingSecret: 'source-private-key', + eventTypes: ['app_mention'], + }), + }) + assert.equal(create.status, 201, await create.clone().text()) + let source = decodeEventSource(await create.json()) + const endpoint = new URL(source.endpointUrl!).pathname + const callback = (payload: unknown, secret = 'source-private-key') => { + const body = JSON.stringify(payload) + const timestamp = String(Math.floor(Date.now() / 1000)) + return app.request(endpoint, { + method: 'POST', + body, + headers: { + 'content-type': 'application/json', + 'x-slack-request-timestamp': timestamp, + 'x-slack-signature': `v0=${createHmac('sha256', secret).update(`v0:${timestamp}:${body}`).digest('hex')}`, + }, + }) + } + assert.equal((await callback({ type: 'url_verification', challenge: 'verified' })).status, 200) + const created = await service.control.flows.createFlow('operator', 'Source acceptance', 'create') + const flowId = created.flow.flowId + const draft = await service.control.drafts.changeDraft('operator', flowId, created.flow.draftRevisionId, [ + { + kind: 'graph.node.create', + nodeId: 'slack', + node: { + kind: 'integration', + name: 'Slack', + connectionId: 'connection', + definition: integrationDefinitions.find((item) => item.snapshot.key === 'slackbot.on_event')!.snapshot, + config: inputValues({ sourceId: source.sourceId, eventTypes: ['app_mention'] }), + }, + }, + ]) + const operation = await service.control.publications.publishFlow('operator', flowId, draft.revision.revisionId, currentEngineContract, null, 'publish') + await service.tickIntegration() + await service.tickMaintenance() + assert.equal(service.control.publications.getPublishOperation('operator', flowId, operation.operationId).status, 'succeeded') + const event = { + type: 'event_callback', + api_app_id: 'A123', + team_id: 'T123', + event_id: 'Ev123', + authorizations: [{ team_id: 'T123', user_id: 'U123', is_bot: true }], + event: { + type: 'app_mention', + user: 'UOTHER', + channel: 'C123', + text: 'hello', + ts: '123.1', + }, + } + let sourceCount = 1 + for (const count of [1, 10, 100]) { + while (sourceCount < count) { + sourceCount++ + const sibling = await app.request('/v1/event-sources', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + kind: 'slack', + version: 1, + name: 'Sibling ' + sourceCount, + connectionId: 'unrelated', + teamId: null, + appId: 'A123', + workspaceId: 'T' + (1000 + sourceCount), + signingSecret: 'source-private-key', + eventTypes: ['app_mention'], + }), + }) + assert.equal(sibling.status, 201, await sibling.clone().text()) + } + callbackCalls = 0 + const started = performance.now() + for (let attempt = 0; attempt < 2; attempt++) assert.equal((await callback({ ...event, event_id: 'Ev' + count })).status, 200) + assert.equal(callbackCalls, 4, 'Each callback must route once and deliver only to its matching Source') + await service.tickIntegration() + console.log( + 'PASS application routing: ' + count + ' Sources, 2 Connector calls/callback, repeated delivery ' + (performance.now() - started).toFixed(2) + ' ms', + ) + } + assert.equal(service.control.runs.listRuns('operator', flowId, 20).page.runs.length, 3) + const update = await app.request(`/v1/event-sources/${source.sourceId}`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + version: 1, + expectedRevision: source.revision, + name: 'Edited Slack', + enabled: true, + eventTypes: source.eventTypes, + signingSecret: 'replacement-private-key', + }), + }) + assert.equal(update.status, 200, await update.clone().text()) + source = decodeEventSource(await update.json()) + assert.equal(source.verifiedAt, null) + assert.equal((await callback({ type: 'url_verification', challenge: 'old' })).status, 401) + assert.equal((await callback({ type: 'url_verification', challenge: 'new' }, 'replacement-private-key')).status, 200) + const local = JSON.stringify(database.connection.prepare('SELECT * FROM event_sources').all()) + assert.ok(!local.includes('source-private-key') && !local.includes('replacement-private-key')) + assert.deepEqual(database.connection.prepare('SELECT * FROM event_source_apps').all(), []) + service.control.flows.retireFlow('operator', flowId) + await service.tickMaintenance() + await service.tickIntegration() + await harness.disconnect() + const removed = await app.request(`/v1/event-sources/${source.sourceId}`, { + method: 'DELETE', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ version: 1, expectedRevision: source.revision }), + }) + assert.equal(removed.status, 200, await removed.clone().text()) + assert.equal(database.connection.prepare('SELECT COUNT(*) AS count FROM event_sources WHERE source_id = ?').get(source.sourceId)?.count, 0) + console.log('PASS Open Flow + Connector: create, signed verification, publish, event deduplication, edit, reverify and deletion after disconnect') +} finally { + await Effect.runPromise(Scope.close(scope, Exit.void)) + database.close() + await harness.close() +} diff --git a/apps/server/scripts/lab/acceptance.ts b/apps/server/scripts/lab/acceptance.ts index 87fc822e..ca4a39bb 100644 --- a/apps/server/scripts/lab/acceptance.ts +++ b/apps/server/scripts/lab/acceptance.ts @@ -25,6 +25,14 @@ export function runtimeAssertions( return errors } if (status != 'completed') return [`Run ${sample.name} ended ${status}.`] + if (id === 'poll-batch') { + const completed = events.find((event) => event.kind === 'run.completed') + const result = completed?.payload.result + if (result == null || typeof result !== 'object' || !('outputs' in result) || !isDeepStrictEqual(result.outputs, { messages: sample.outputs?.events })) + errors.push('Return the complete batch of messages supplied to this Run.') + if (calls.length || modelCalls.length) errors.push('No external requests are needed.') + return errors + } if (id == 'wait-auto-continue') { const resolved = events.filter((event) => event.kind == 'run.resolved') const completed = events.find((event) => event.kind == 'run.completed') diff --git a/apps/server/scripts/lab/mocks.ts b/apps/server/scripts/lab/mocks.ts index 1602cdb6..6d371340 100644 --- a/apps/server/scripts/lab/mocks.ts +++ b/apps/server/scripts/lab/mocks.ts @@ -109,6 +109,7 @@ export const actions: ConnectorActionMetadata[] = [ }, ] export const accounts: ConnectorConnection[] = [ + { connectionId: 'mail-test', displayName: 'Mail test account', serviceId: 'gmail', status: 'active', isDefault: true }, { connectionId: 'marketplace:oomol:lab-data', displayName: 'Data test account', serviceId: 'lab-data', status: 'active', isDefault: true }, { connectionId: 'orders-test', displayName: 'Orders test account', serviceId: 'lab-orders', status: 'active', isDefault: false }, { connectionId: 'orders-production', displayName: 'Orders production account', serviceId: 'lab-orders', status: 'active', isDefault: true }, diff --git a/apps/server/scripts/lab/reference.ts b/apps/server/scripts/lab/reference.ts index 79e46775..06c9044a 100644 --- a/apps/server/scripts/lab/reference.ts +++ b/apps/server/scripts/lab/reference.ts @@ -31,6 +31,29 @@ export async function reference(call: ToolCall, id: string, flowId: string, stag } const dataPort = { schema: {}, nullable: true }, stringPort = { schema: { type: 'string' }, nullable: false } + if (id === 'poll-batch') { + await call('flow_schema', { type: 'poll' }) + const before = await read() + const created = await submit(before.revision, [ + { op: 'node.add', type: 'poll', as: 'mail', name: 'Mail', config: { key: 'gmail.on_message_received', connectionId: 'mail-test' } }, + { + op: 'node.add', + type: 'outputs', + as: 'result', + name: 'Result', + config: { outputDefs: { messages: { schema: { type: 'array', items: { type: 'object' } }, nullable: false } } }, + inputs: { messages: { kind: 'output', node: '$mail', port: 'events' } }, + }, + { op: 'edge.connect', source: '$mail', target: '$result' }, + ]) + const refs = created.nodes as Record + await read([refs.mail!]) + const renamed = await submit(created.revision, [{ op: 'node.update', node: refs.mail!, set: { name: 'Mail pending' } }]) + await submit(renamed.revision, [{ op: 'node.update', node: refs.mail!, set: { name: 'Mail events' } }]) + await read([refs.mail!]) + await call('flow_check', { flowId, revisionId: savedRevision }) + return + } if (id == 'wait-auto-continue') { await call('flow_schema', { type: 'wait' }) const before = await read() diff --git a/apps/server/scripts/lab/scenarios.ts b/apps/server/scripts/lab/scenarios.ts index 5dd2cf32..dd68de4b 100644 --- a/apps/server/scripts/lab/scenarios.ts +++ b/apps/server/scripts/lab/scenarios.ts @@ -1,10 +1,19 @@ import type { ChangeOperation, JsonValue, ManagedTaskDefinition, RevisionContent } from '@oomol-lab/open-flow/flow-change' import { applyFlowChanges, currentFlowModelVersion } from '@oomol-lab/open-flow/flow-change' +import { triggerConfigValues } from '@oomol-lab/open-flow/poll-trigger' +import { triggerDefinitions } from '@oomol-lab/open-flow/provider-triggers' import { isDeepStrictEqual } from 'node:util' import { fulfillmentSamples, fulfillmentStages } from './fulfillment.ts' export const scenarios = [ + { + id: 'poll-batch', + entry: { kind: 'poll' }, + name: 'Author a batch Poll', + version: 1, + task: 'Create a Gmail Poll with Mail test account and default Gmail settings, read its events output, change only its name, and connect it to Outputs returning the complete messages array from events. Preserve the account and configuration. Check the Flow. Each deterministic draft Run receives a batch of messages.', + }, { id: 'webhook-response-node', entry: { kind: 'webhook' }, @@ -185,6 +194,7 @@ function source(nodeId: string, handle: string, from: string, output: string): C } export function initialOperations(id: string): ChangeOperation[] { if ( + id == 'poll-batch' || id == 'wait-auto-continue' || id == 'node-errors' || id == 'node-retries' || @@ -361,6 +371,24 @@ export interface Sample { parameters?: Record } export function samples(id: string, stage = 0): Sample[] { + if (id === 'poll-batch') + return ['First', 'Second'].map((name) => ({ + name, + orders: [], + order, + outputs: { + events: [1, 2].map((index) => ({ + historyId: null, + labelIds: [], + messageId: `${name}-${index}`, + messageTimestamp: '2026-10-27T00:00:00Z', + sender: 'sender@example.test', + subject: `${name}-${index}`, + threadId: `${name}-${index}`, + to: 'reader@example.test', + })), + }, + })) if (id == 'node-errors') return [{ name: 'handled-and-successful', orders: [], order }] if (id == 'node-retries') return [{ name: 'configured-and-cleared', orders: [], order }] if (id == 'workflow-call') return [0, -2.5, 12.5].map((amount) => ({ name: String(amount), orders: [], order, outputs: { amount } })) @@ -428,6 +456,20 @@ function effectiveNode(content: RevisionContent, id: string): unknown { } /** Independent assertions: compare behavior and preserved effective configurations, not generated IDs. */ export function assertions(id: string, base: RevisionContent, current: RevisionContent): string[] { + if (id === 'poll-batch') { + const poll = Object.values(current.document.graph.nodes).find((candidate) => candidate.kind === 'poll') + const expected = triggerDefinitions.find((definition) => definition.snapshot.key === 'gmail.on_message_received')!.snapshot + return poll?.kind === 'poll' && + poll.name === 'Mail events' && + poll.connectionId === 'mail-test' && + poll.definition.definitionVersion === 2 && + poll.definition.outputs[0]?.handle === 'events' && + poll.definition.key === expected.key && + isDeepStrictEqual(triggerConfigValues(expected.configInputs, poll.config), triggerConfigValues(expected.configInputs, {})) + ? [] + : ['Preserve the selected Gmail account, default configuration and batch Poll contract.'] + } + if (id == 'wait-auto-continue') { const nodes = Object.values(current.document.graph.nodes) const relative = nodes.find((candidate) => candidate.name == 'Relative wait updated') diff --git a/apps/server/scripts/lab/session.ts b/apps/server/scripts/lab/session.ts index 6663442f..5c60506e 100644 --- a/apps/server/scripts/lab/session.ts +++ b/apps/server/scripts/lab/session.ts @@ -6,6 +6,7 @@ import { serve } from '@hono/node-server' import { isLabCliOffline } from '@oomol-lab/open-flow-command/lab' import { authoringNode } from '@oomol-lab/open-flow/control-requests' import { validateFlowInputs } from '@oomol-lab/open-flow/flow-semantics' +import { triggerDefinitions } from '@oomol-lab/open-flow/provider-triggers' import { currentEngineContract } from '@oomol-lab/open-flow/runtime-contract' import * as Effect from 'effect/Effect' import * as Exit from 'effect/Exit' @@ -284,7 +285,7 @@ export class LabSession { ServerService.open(this.database, { capabilities: { connector: () => this.mocks.connector, llm: () => createLlm(this.modelOrigin, 'lab-only') }, runtime: { runTimeoutMs: 15_000 }, - triggerDefinitions: [], + triggerDefinitions: this.scenarioId === 'poll-batch' ? triggerDefinitions.filter((item) => item.snapshot.key === 'gmail.on_message_received') : [], }).pipe(Scope.provide(this.scope)), ) const operator = new OperatorSession(new OperatorStore(this.database), this.token, false) @@ -394,7 +395,8 @@ export class LabSession { const triggers = Object.entries(revision.content.document.graph.nodes).filter(([id, node]) => 'node' in entry ? id == entry.node : node.kind == entry.kind, ) - if (triggers.length != 1 || triggers[0]![1].kind != 'start') errors.push('The scenario entry must resolve to one Start node.') + if (triggers.length != 1 || (triggers[0]![1].kind != 'start' && this.scenarioId !== 'poll-batch')) + errors.push('The scenario entry must resolve to one Start node.') else for (const sample of samples(this.scenarioId, this.stage)) { Object.assign(this.mocks.sample, sample) diff --git a/apps/server/test/connectorClient.test.ts b/apps/server/test/connectorClient.test.ts index ea5d7529..432a3775 100644 --- a/apps/server/test/connectorClient.test.ts +++ b/apps/server/test/connectorClient.test.ts @@ -1421,7 +1421,9 @@ it('routes a registered Trigger request separately from generic proxy', async () await expect( client.trigger('example', 'connection-work', 'example.on_event', { operation: 'read', config: {}, checkpoint: null }, new AbortController().signal), ).resolves.toEqual({ checkpoint: null, events: [] }) - expect(requests).toEqual([{ path: '/v1/providers/example/triggers/example.on_event/execute', body: { operation: 'read', config: {}, checkpoint: null } }]) + expect(requests).toEqual([ + { path: '/v1/providers/example/triggers/example.on_event/execute?appId=connection-work', body: { operation: 'read', config: {}, checkpoint: null } }, + ]) }) it.each([ @@ -1442,3 +1444,55 @@ it.each([ client.trigger('example', 'connection-work', 'example.on_event', { operation: 'read', config: {}, checkpoint: null }, new AbortController().signal), ).rejects.toMatchObject({ code }) }) + +it('deletes a Source as an operator without requiring a surviving connection and rejects other callers', async () => { + const requests: string[] = [] + const origin = await startConnector(async (request, response) => { + requests.push(request.url!) + send(response, 200, { success: true, data: { deleted: true } }) + }) + const client = new ConnectorClient(origin, 'runtime-token') + const request = { + operation: 'sourceReference' as const, + request: { action: 'delete' as const, deploymentId: 'deployment', reference: { applicationId: 'app', sourceId: 'source' } }, + } + await expect(client.trigger('slackbot', 'removed', 'slackbot.on_event', request, new AbortController().signal)).rejects.toMatchObject({ + code: 'connector.access-invalid', + }) + expect(requests).toEqual([]) + await expect( + client.trigger('slackbot', 'removed', 'slackbot.on_event', request, new AbortController().signal, { + scope: 'catalog', + providerAccess: { mode: 'implicit', version: 1, accessRevision: 0, bindings: [], sharedAccessDigest: 'implicit' }, + purpose: 'catalog', + source: 'operator', + }), + ).resolves.toEqual({ deleted: true }) + expect(requests).toEqual(['/v1/providers/slackbot/triggers/slackbot.on_event/execute?appId=removed']) +}) + +it('preserves a Source revision conflict for the control plane', async () => { + const origin = await startConnector(async (request, response) => { + if (request.url === '/v1/apps') return send(response, 200, { data: [app], success: true }) + send(response, 409, { errorCode: 'invalid_input', errorMessage: 'The Source changed' }) + }) + const client = new ConnectorClient(origin, 'runtime-token') + await expect( + client.trigger( + 'example', + 'connection-work', + 'example.on_event', + { + operation: 'sourceReference', + request: { + action: 'update', + deploymentId: 'deployment', + reference: { applicationId: 'app', sourceId: 'source' }, + expectedRevision: 1, + input: { eventTypes: ['app_mention'] }, + }, + }, + new AbortController().signal, + ), + ).rejects.toMatchObject({ code: 'connector.input-invalid', status: 409 }) +}) diff --git a/apps/server/test/controlApiConformance.test.ts b/apps/server/test/controlApiConformance.test.ts index 9b6b1ec5..c364494c 100644 --- a/apps/server/test/controlApiConformance.test.ts +++ b/apps/server/test/controlApiConformance.test.ts @@ -37,7 +37,8 @@ import { Store } from '../node/storage/store.ts' import { createServerApp } from '../node/transport/http.ts' import { createConnectorHost } from './connectorHost.ts' import { closeService, openService, startService } from './serviceFixture.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' +const sourceProtocolFixture = sourceReferenceFixture() const connectorProvider: ConnectorProvider = { icon: 'https://connector.example/icons/mail.svg', @@ -118,7 +119,7 @@ async function createHarness( ? [connectorConnection] : [], trigger: async (_provider, _connection, _trigger, request) => - request.operation === 'source' + request.operation === 'sourceReference' ? sourceProtocolFixture(request.request, eventConnection.providerAccountId) : [{ value: teamId, label: 'Engineering (ENG)' }], listProviders: async () => [connectorProvider], @@ -745,7 +746,7 @@ it('Server event source Team scope and authorization page conformance', async () ]) vi.spyOn(connector, 'listConnections').mockResolvedValue([eventConnection]) vi.spyOn(connector, 'trigger').mockImplementation(async (_provider, _connection, _trigger, request) => - request.operation === 'source' ? sourceProtocolFixture(request.request, eventConnection.providerAccountId) : undefined, + request.operation === 'sourceReference' ? sourceProtocolFixture(request.request, eventConnection.providerAccountId) : undefined, ) const service = await openService(':memory:', { capabilities: { connector: () => connector } }) const app = createServerApp(service, { resolveControlActor: () => 'operator' }) diff --git a/apps/server/test/eventSourceList.test.ts b/apps/server/test/eventSourceList.test.ts index 2ffddcae..f05afb69 100644 --- a/apps/server/test/eventSourceList.test.ts +++ b/apps/server/test/eventSourceList.test.ts @@ -1,7 +1,7 @@ import type { ConnectorHost } from '../node/deployment/connector-contract.ts' import { decodeEventSources } from '@oomol-lab/open-flow/control-api' -import { sourceResultSchema } from '@oomol-lab/open-flow/event-source' +import { sourceReferenceResultSchema } from '@oomol-lab/open-flow/event-source' import { expect, it, onTestFinished, vi } from 'vitest' import { ZodError } from 'zod' import { runEnvironment } from '../node/application/run-environment.ts' @@ -11,7 +11,9 @@ import { isolatedVmEngineDigest } from '../node/runtime/isolated-vm.ts' import { Database } from '../node/storage/database.ts' import { Store } from '../node/storage/store.ts' import { createConnectorHost } from './connectorHost.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' + +const respond = sourceReferenceFixture() function fixture(trigger: ConnectorHost['trigger']) { const database = Database.open(':memory:') @@ -26,19 +28,24 @@ function fixture(trigger: ConnectorHost['trigger']) { Date.now, () => {}, ) - function create(name: string, projected = false) { + function create(name: string, projected = false, teamId: string | null = null) { const input = { version: 1 as const, name, connectionId: name, - teamId: null, + teamId, verificationToken: 'secret', encryptKey: 'secret', eventTypes: ['im.message.receive_v1'], manageSubscriptions: false, } - const { source } = sourceResultSchema.parse(sourceProtocolFixture({ action: 'initialize', input }, 'cli_' + name)) - return store.eventSources.create(input, 'feishu', { ...source, state: projected ? source.state : {} }) + const reserved = store.eventSources.reserve(input, name) + const { source } = sourceReferenceResultSchema.parse( + respond({ action: 'create', deploymentId: reserved.connector!.deploymentId, requestKey: reserved.connector!.requestKey, input }, 'cli_' + name), + ) + store.eventSources.saveReference(reserved, source, 'create') + if (!projected) database.connection.prepare("UPDATE event_sources SET state_json = '{}' WHERE source_id = ?").run(reserved.sourceId) + return store.eventSources.require(reserved.sourceId) } return { database, store, runtime, create } } @@ -46,15 +53,15 @@ function fixture(trigger: ConnectorHost['trigger']) { it.each(['connector', 'kind', 'projection', 'application'])('isolates a %s failure from a healthy source', async (failure) => { const context = fixture(async (_provider, connection, _trigger, request) => { if (connection === 'bad') throw new Error('Connector unavailable') - if (request.operation !== 'source') throw new Error('Unexpected operation') - return sourceProtocolFixture(request.request) + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + return respond(request.request) }) const bad = context.create('bad', failure !== 'connector' && failure !== 'kind') const good = context.create('good') if (failure === 'kind') context.database.connection.prepare("UPDATE event_sources SET kind = 'unknown' WHERE source_id = ?").run(bad.sourceId) if (failure === 'projection') context.database.connection.prepare('UPDATE event_sources SET state_json = \'{"projection":{}}\' WHERE source_id = ?').run(bad.sourceId) - if (failure === 'application') context.database.connection.prepare("UPDATE event_sources SET kind = 'slack' WHERE source_id = ?").run(bad.sourceId) + if (failure === 'application') context.database.connection.prepare('UPDATE event_sources SET connector_json = NULL WHERE source_id = ?').run(bad.sourceId) const result = decodeEventSources(await context.runtime.list()) expect(result.sources).toHaveLength(2) expect(result.sources.find((source) => source.sourceId === bad.sourceId)).toMatchObject({ appId: null, lastError: expect.any(String) }) @@ -85,7 +92,7 @@ it.each( lastError: 'Stored source details are invalid or unsupported. Repair or recreate this event source.', }) expect(result.sources.find((source) => source.sourceId === good.sourceId)).toMatchObject({ appId: 'cli_good', name: 'good' }) - expect(inspect).not.toHaveBeenCalled() + expect(inspect).toHaveBeenCalledTimes(1) }) it.each(['caller', 'budget'])('bounds concurrent inspection and stops queued work on %s cancellation', async (cancellation) => { @@ -94,8 +101,8 @@ it.each(['caller', 'budget'])('bounds concurrent inspection and stops queued wor const context = fixture(async (_provider, _connection, _trigger, request, signal) => { calls.push(signal) await pending.promise - if (request.operation !== 'source') throw new Error('Unexpected operation') - return sourceProtocolFixture(request.request) + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + return respond(request.request) }) for (let index = 0; index < 8; index++) context.create('source' + index) const controller = new AbortController() @@ -113,7 +120,7 @@ it.each(['caller', 'budget'])('bounds concurrent inspection and stops queued wor pending.resolve() await new Promise((resolve) => setImmediate(resolve)) expect(calls).toHaveLength(4) - expect(context.store.eventSources.unprojected()).toHaveLength(8) + expect(context.store.eventSources.ids()).toHaveLength(8) }) it('continues queued inspection within the concurrency limit', async () => { @@ -125,8 +132,8 @@ it('continues queued inspection within the concurrency limit', async () => { maximum = Math.max(maximum, active) await first.promise active-- - if (request.operation !== 'source') throw new Error('Unexpected operation') - return sourceProtocolFixture(request.request) + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + return respond(request.request) }) satisfies NonNullable const context = fixture(inspect) for (let index = 0; index < 9; index++) context.create('source' + index) @@ -137,3 +144,19 @@ it('continues queued inspection within the concurrency limit', async () => { expect(inspect).toHaveBeenCalledTimes(9) expect(maximum).toBe(4) }) + +it('refreshes and lists only the requested team, including the deployment scope', async () => { + const inspect = vi.fn(async (_provider, _connection, _trigger, request) => { + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + return respond(request.request) + }) satisfies NonNullable + const context = fixture(inspect) + context.create('deployment', true) + context.create('alpha', true, 'team-a') + context.create('beta', true, 'team-b') + expect((await context.runtime.list('team-a')).sources.map((source) => source.name)).toEqual(['alpha']) + expect(inspect.mock.calls.map((call) => call[1])).toEqual(['alpha']) + inspect.mockClear() + expect((await context.runtime.list(null)).sources.map((source) => source.name)).toEqual(['deployment']) + expect(inspect.mock.calls.map((call) => call[1])).toEqual(['deployment']) +}) diff --git a/apps/server/test/eventSourceState.test.ts b/apps/server/test/eventSourceState.test.ts index 9ec26bba..568a64b2 100644 --- a/apps/server/test/eventSourceState.test.ts +++ b/apps/server/test/eventSourceState.test.ts @@ -4,39 +4,43 @@ import { isolatedVmEngineDigest } from '../node/runtime/isolated-vm.ts' import { Database } from '../node/storage/database.ts' import { Store } from '../node/storage/store.ts' -it('rolls back application verification when source state persistence fails', () => { +it('rolls back reference projection persistence and rejects stale local revisions', () => { const database = Database.open(':memory:') try { const store = new Store(database, runEnvironment(isolatedVmEngineDigest), Date.now) - const source = store.eventSources.create( + const source = store.eventSources.reserve( { - kind: 'slack', + kind: 'zoom', version: 1, - name: 'Slack', + name: 'Zoom', connectionId: 'connection', teamId: null, appId: 'A1', - workspaceId: 'T1', signingSecret: 'secret', - eventTypes: ['app_mention'], - }, - 'slack', - { - identity: 'workspace', - config: { appId: 'A1' }, - secrets: { signingSecret: 'secret' }, - state: {}, - eventTypes: ['app_mention'], + eventTypes: ['recording.completed'], }, + 'create', ) + const snapshot = { + reference: { applicationId: 'app', sourceId: 'remote' }, + revision: 1, + eventTypes: source.eventTypes, + ready: true, + view: { + appId: 'A1', + manageSubscriptions: false, + verificationTokenConfigured: false, + encryptKeyConfigured: false, + verifiedAt: new Date(100).toISOString(), + }, + } database.connection.exec("CREATE TRIGGER reject_source_state BEFORE UPDATE OF state_json ON event_sources BEGIN SELECT RAISE(ABORT, 'write failed'); END") - expect(() => store.eventSources.saveState(source, { verifiedAt: 100 })).toThrow('write failed') - expect(database.connection.prepare('SELECT verified_at FROM event_source_apps').get()).toEqual({ verified_at: null }) + expect(() => store.eventSources.saveReference(source, snapshot, 'create')).toThrow('write failed') + expect(store.eventSources.require(source.sourceId).connector?.reference).toBeUndefined() database.connection.exec('DROP TRIGGER reject_source_state') - expect(store.eventSources.saveState(source, { verifiedAt: 100 })).toBe(true) - expect(database.connection.prepare('SELECT verified_at FROM event_source_apps').get()).toEqual({ verified_at: 100 }) - expect(store.eventSources.saveState({ ...source, revision: source.revision + 1 }, { verifiedAt: 200 })).toBe(false) - expect(database.connection.prepare('SELECT verified_at FROM event_source_apps').get()).toEqual({ verified_at: 100 }) + expect(store.eventSources.saveReference(source, snapshot, 'create')).toBe(true) + expect(store.eventSources.saveReference({ ...source, revision: source.revision + 1 }, snapshot)).toBe(false) + expect(database.connection.prepare('SELECT * FROM event_source_apps').all()).toEqual([]) } finally { database.close() } diff --git a/apps/server/test/eventSources.test.ts b/apps/server/test/eventSources.test.ts index 5025593e..be1ad782 100644 --- a/apps/server/test/eventSources.test.ts +++ b/apps/server/test/eventSources.test.ts @@ -14,7 +14,8 @@ import { ConnectorClient } from '../node/deployment/connector.ts' import { createServerApp } from '../node/transport/http.ts' import { createConnectorHost } from './connectorHost.ts' import { closeService, openService } from './serviceFixture.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' +const sourceProtocolFixture = sourceReferenceFixture() const now = Date.parse('2026-09-14T08:00:00.000Z') const definition = integrationDefinitions.find((item) => item.snapshot.key == 'feishu_app_bot.on_event')! @@ -33,7 +34,7 @@ function delivery(result: unknown) { return { method: 'POST', body: JSON.stringify(result), headers: { 'content-type': 'application/json' } } } -async function setup() { +async function setup(dropResourceResponse = false) { const requests: string[] = [] const connector = createConnectorHost({ listConnections: async () => [ @@ -47,10 +48,14 @@ async function setup() { }, ], trigger: async (_provider, _connection, _trigger, request) => { - if (request.operation === 'source') return sourceProtocolFixture(request.request) - if (request.operation !== 'resource') throw new Error('Unexpected operation') - requests.push(`${request.active ? 'subscribe' : 'unsubscribe'} ${JSON.stringify(request.config.resource)}`) - return { outcome: 'ready' } + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + if (request.request.action === 'resource') requests.push(`${request.request.active ? 'subscribe' : 'unsubscribe'} ${request.request.key}`) + const result = sourceProtocolFixture(request.request) + if (dropResourceResponse && request.request.action === 'resource') { + dropResourceResponse = false + throw new Error('Resource response lost') + } + return result }, }) const service = await openService(':memory:', { @@ -191,7 +196,8 @@ it('creates an application event source without querying tenant information', as listConnections: async () => [ { connectionId: input.connectionId, providerAccountId: 'cli_test', displayName: 'Feishu', serviceId: provider, status: 'active', isDefault: true }, ], - trigger: async (_provider, _connection, _trigger, request) => (request.operation === 'source' ? sourceProtocolFixture(request.request) : undefined), + trigger: async (_provider, _connection, _trigger, request) => + request.operation === 'sourceReference' ? sourceProtocolFixture(request.request) : undefined, proxy: vi.fn(async () => { throw new Error('Creating an event source must not require a provider API request.') }), @@ -216,7 +222,8 @@ it('rejects missing application identity instead of accepting a manually supplie listConnections: async () => [ { connectionId: input.connectionId, displayName: 'Unknown app', serviceId: 'feishu_app_bot', status: 'active', isDefault: true }, ], - trigger: async (_provider, _connection, _trigger, request) => (request.operation === 'source' ? sourceProtocolFixture(request.request, '') : undefined), + trigger: async (_provider, _connection, _trigger, request) => + request.operation === 'sourceReference' ? sourceProtocolFixture(request.request, '') : undefined, proxy: async () => { throw new Error('Identity must be checked before querying the provider.') }, @@ -228,7 +235,9 @@ it('rejects missing application identity instead of accepting a manually supplie expect((await request(input)).status).toBe(409) expect((await request({ ...input, appId: 'cli_forged' })).status).toBe(400) expect((await request({ ...input, provider: 'feishu' })).status).toBe(400) - expect((await (await app.request('/v1/event-sources')).json()).sources).toEqual([]) + expect((await (await app.request('/v1/event-sources')).json()).sources).toMatchObject([ + { appId: null, verifiedAt: null, lastError: expect.stringContaining('unfinished') }, + ]) }) it('returns the Flow team even when it has no event sources and rejects unknown Flows', async () => { @@ -282,15 +291,14 @@ it.each(['normal', 'failure', 'update'])('hydrates persisted sources without los { connectionId: input.connectionId, providerAccountId: 'cli_test', displayName: 'App', serviceId: 'feishu_app_bot', status: 'active', isDefault: true }, ], trigger: async (_provider, _connection, _trigger, request) => { - if (request.operation !== 'source') throw new Error('Unexpected operation') + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') actions.push(request.request.action) - if (request.request.action === 'inspect' && failFirst && actions.filter((action) => action === 'inspect').length === 1) - throw new Error('Connector unavailable') - if (request.request.action === 'inspect' && scenario === 'update') { + if (request.request.action === 'get' && failFirst && actions.filter((action) => action === 'get').length === 1) throw new Error('Connector unavailable') + if (request.request.action === 'get' && scenario === 'update' && actions.filter((action) => action === 'get').length === 1) { const database = new DatabaseSync(file) database .prepare("UPDATE event_sources SET revision = revision + 1, name = 'Concurrent edit', state_json = ?") - .run(JSON.stringify((sourceProtocolFixture(request.request) as { source: { state: unknown } }).source.state)) + .run(JSON.stringify({ projection: (sourceProtocolFixture(request.request) as { source: unknown }).source })) database.close() } return sourceProtocolFixture(request.request) @@ -328,7 +336,7 @@ it.each(['normal', 'failure', 'update'])('hydrates persisted sources without los endpointUrl: original.endpointUrl, }) } - expect(actions).toEqual(failFirst ? ['initialize', 'inspect', 'inspect'] : ['initialize', 'inspect']) + expect(actions[0]).toBe('create') } finally { await closeService(service) await rm(directory, { recursive: true, force: true }) @@ -340,3 +348,42 @@ it.each(['feishu', 'fireflies', 'jira', 'unknown'])('rejects application ingress const response = await app.request(`/v1/event-source-apps/${kind}/cli_test/events`, delivery({ verify: true })) expect(response.status).toBe(404) }) + +it('retries an uncertain Feishu resource through Connector ownership instead of blocking publication', async () => { + const context = await setup(true) + const { service, source } = context + try { + const created = await service.control.flows.createFlow('operator', 'Recover resource', 'recover-resource') + const flowId = created.flow.flowId + const draft = await service.control.drafts.changeDraft('operator', flowId, created.flow.draftRevisionId, [ + { + kind: 'graph.node.create', + nodeId: 'feishu', + node: { + kind: 'integration', + name: 'Feishu', + connectionId: input.connectionId, + definition: definition.snapshot, + config: inputValues({ sourceId: source.sourceId, eventTypes: ['approval_instance'], resource: { kind: 'approval', id: 'approval' } }), + }, + }, + ]) + const operation = await service.control.publications.publishFlow( + 'operator', + flowId, + draft.revision.revisionId, + currentEngineContract, + null, + 'recover-publication', + ) + await service.tickIntegration() + await service.tickMaintenance() + expect(service.control.publications.getPublishOperation('operator', flowId, operation.operationId).status).not.toBe('succeeded') + await service.tickIntegration(new Date(now + 60_000).toISOString()) + await service.tickMaintenance() + expect(service.control.publications.getPublishOperation('operator', flowId, operation.operationId).status).toBe('succeeded') + expect(context.requests.filter((request) => request.startsWith('subscribe '))).toHaveLength(2) + } finally { + await closeService(service) + } +}) diff --git a/apps/server/test/lab-acceptance.test.ts b/apps/server/test/lab-acceptance.test.ts index ec8c130b..b35247e2 100644 --- a/apps/server/test/lab-acceptance.test.ts +++ b/apps/server/test/lab-acceptance.test.ts @@ -1,9 +1,11 @@ import type { RunEvent } from '@oomol-lab/open-flow/control-api' import type { RevisionContent } from '@oomol-lab/open-flow/flow-change' -import { currentFlowModelVersion } from '@oomol-lab/open-flow/flow-change' +import { currentFlowModelVersion, inputValues } from '@oomol-lab/open-flow/flow-change' +import { triggerDefinitions } from '@oomol-lab/open-flow/provider-triggers' import { expect, it } from 'vitest' import { runtimeAssertions } from '../scripts/lab/acceptance.ts' +import { assertions, emptyContent } from '../scripts/lab/scenarios.ts' const task = (name: string) => ({ kind: 'task' as const, name, inputs: {}, task: { name, moduleId: 'code', inputs: [], outputs: [], capabilities: [] } }) const content: RevisionContent = { @@ -43,3 +45,34 @@ it('rejects external calls even when both tasks complete correctly', () => { expect(verify(events, [{ action: 'unexpected', connection: 'account', input: {}, invocationId: 'unexpected' }])).not.toEqual([]) expect(verify(events, [], [{}])).not.toEqual([]) }) + +it.each([ + [{}, true], + [{ search: '', maxMessagesPerPoll: 25, includeSpamAndTrash: false }, true], + [{ search: 'from:someone@example.test' }, false], + [{ maxMessagesPerPoll: 1 }, false], + [{ includeSpamAndTrash: true }, false], +])('checks effective Gmail configuration in the Poll Lab: %j', (config, accepted) => { + const definition = triggerDefinitions.find((item) => item.snapshot.key === 'gmail.on_message_received')!.snapshot + if (definition.type !== 'poll') throw new Error('Expected Gmail Poll definition') + const current: RevisionContent = { + ...emptyContent, + document: { + ...emptyContent.document, + graph: { + edges: [], + nodes: { + mail: { + kind: 'poll', + name: 'Mail events', + connectionId: 'mail-test', + definition, + config: inputValues(config), + pollTimes: [{ type: 'every', unit: 'minute', value: 1 }], + }, + }, + }, + }, + } + expect(assertions('poll-batch', emptyContent, current).length === 0).toBe(accepted) +}) diff --git a/apps/server/test/managedEventSources.test.ts b/apps/server/test/managedEventSources.test.ts index 2a6308e3..bf39dd99 100644 --- a/apps/server/test/managedEventSources.test.ts +++ b/apps/server/test/managedEventSources.test.ts @@ -8,7 +8,7 @@ import { ConnectorTaskError } from '../node/deployment/connector-contract.ts' import { createServerApp } from '../node/transport/http.ts' import { createConnectorHost } from './connectorHost.ts' import { openService } from './serviceFixture.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' const now = 1_800_000_000_000 function delivery(id: string) { @@ -28,44 +28,28 @@ it('shares a managed Jira project subscription and deduplicates fanout without e let clock = now let failVerification = false let failRenewal = false + const respond = sourceReferenceFixture() const calls: { active: boolean; key: string }[] = [] const connector = createConnectorHost({ trigger: async (_provider, connectionId, _trigger, request) => { const id = connectionId.slice(-1) - if (request.operation === 'resource') { - if (failRenewal && request.active) throw new ConnectorTaskError('connector.unavailable', 'Temporary subscription failure') - calls.push({ active: request.active, key: request.requestKey }) - return { outcome: 'ready', subscription: request.active ? { webhookId: 42, expiresAt: now + 30 * 86400000 } : {} } - } - if (request.operation === 'source') { + if (request.operation === 'sourceReference') { if (request.request.action === 'receive' && failVerification) throw new ConnectorTaskError('connector.unavailable', 'Temporary verification failure') - const result = sourceProtocolFixture(request.request, `jira:cloud${id}:user`, clock) as { source?: { state: Record }; event?: unknown } - if (request.request.action === 'prepare') - return { ...result, subscriptions: [{ key: 'project', config: { projectId: '100', endpointUrl: request.request.endpointUrl } }] } - if (request.request.action === 'inspect') { - const state = request.request.source.state - return { - source: { - ...request.request.source, - state: { - ...state, - projection: { - ready: state.resourceStatus === 'ready', - expiresAt: now + 30 * 86400000, - view: { - appId: `cloud${id}`, - manageSubscriptions: true, - verificationTokenConfigured: false, - encryptKeyConfigured: false, - verifiedAt: state.verifiedAt == null ? null : new Date(Number(state.verifiedAt)).toISOString(), - subscriptionStatus: state.resourceStatus, - lastError: state.lastError, - }, - }, - }, - }, + if (request.request.action === 'resource') { + if (failRenewal && request.request.active) { + const failed = respond({ ...request.request, action: 'get' }) as { source: import('@oomol-lab/open-flow/event-source').SourceReferenceSnapshot } + failed.source.view.subscriptionStatus = 'error' + failed.source.ready = false + throw new ConnectorTaskError('connector.unavailable', 'Temporary subscription failure') } + calls.push({ active: request.request.active, key: `${request.request.reference.sourceId}:${request.request.key}` }) + } + const result = respond(request.request, `jira:cloud${id}:user`) as { source?: import('@oomol-lab/open-flow/event-source').SourceReferenceSnapshot } + if (result.source) { + result.source.view.appId = `cloud${id}` + result.source.expiresAt = now + 30 * 86400000 } + if (request.request.action === 'prepare') return { ...result, subscriptions: [{ key: 'project' }] } return result } throw new Error('Unexpected operation') diff --git a/apps/server/test/mcp.test.ts b/apps/server/test/mcp.test.ts index bf265a80..16ebaf48 100644 --- a/apps/server/test/mcp.test.ts +++ b/apps/server/test/mcp.test.ts @@ -18,7 +18,8 @@ import { Store } from '../node/storage/store.ts' import { createServerApp } from '../node/transport/http.ts' import { createConnectorHost } from './connectorHost.ts' import { closeService, openService, startService } from './serviceFixture.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' +const sourceProtocolFixture = sourceReferenceFixture() const token = 'mcp-test-operator-token-000000000001' const version = '2026-07-28' @@ -182,7 +183,8 @@ it('shares current online details between REST and MCP without exposing historic it('discovers independent Feishu event sources without exposing secrets', async () => { const connector = createConnectorHost({ - trigger: async (_provider, _connection, _trigger, request) => (request.operation === 'source' ? sourceProtocolFixture(request.request) : undefined), + trigger: async (_provider, _connection, _trigger, request) => + request.operation === 'sourceReference' ? sourceProtocolFixture(request.request) : undefined, listConnections: async () => [ { connectionId: 'connection-1', diff --git a/apps/server/test/migrate.test.ts b/apps/server/test/migrate.test.ts index 1ff70d90..022b012b 100644 --- a/apps/server/test/migrate.test.ts +++ b/apps/server/test/migrate.test.ts @@ -160,7 +160,7 @@ it('accepts databases already rebuilt with the new column names and snapshot for ) database.close() const upgraded = Database.open(file) - expect(version(upgraded.connection)).toBe(53) + expect(version(upgraded.connection)).toBe(54) expect(upgraded.connection.prepare('PRAGMA integrity_check').get()).toEqual({ integrity_check: 'ok' }) upgraded.close() }) @@ -211,7 +211,7 @@ it('backfills Revision metadata needed after old content is pruned', async () => const upgraded = Database.open(file) try { - expect(version(upgraded.connection)).toBe(53) + expect(version(upgraded.connection)).toBe(54) expect(upgraded.connection.prepare('SELECT digest, model_version AS modelVersion FROM flow_revisions WHERE revision_id = ?').get('revision')).toEqual({ digest: 'digest', modelVersion: 3, @@ -267,7 +267,7 @@ it('applies the Flow-first schema without foreign keys', async () => { Database.open(file).close() const database = new DatabaseSync(file) try { - expect(version(database)).toBe(53) + expect(version(database)).toBe(54) const tables = database.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name").all() as { readonly name: string }[] @@ -309,7 +309,7 @@ it('upgrades a version 1 Flow database without changing its data', async () => { const reopened = new DatabaseSync(file) try { - expect(version(reopened)).toBe(53) + expect(version(reopened)).toBe(54) expect(reopened.prepare('SELECT revision_id AS revisionId FROM revisions').all()).toEqual([{ revisionId: 'revision-a' }]) expect(reopened.prepare('SELECT name FROM variables').all()).toEqual([]) } finally { @@ -339,7 +339,7 @@ it('adds an immutable Connector Team binding to every existing Flow', async () = const reopened = new DatabaseSync(file) try { - expect(version(reopened)).toBe(53) + expect(version(reopened)).toBe(54) expect(reopened.prepare('SELECT flow_id AS flowId, team_id AS teamId FROM flow_connector_teams').all()).toEqual([{ flowId: 'flow-a', teamId: null }]) expect(reopened.prepare("SELECT name FROM pragma_table_info('runs') WHERE name = 'connector_team_id'").get()).toEqual({ name: 'connector_team_id' }) } finally { @@ -387,13 +387,13 @@ it('rejects a newer Flow schema version without modifying it', async () => { const file = await databaseFile() Database.open(file).close() const database = new DatabaseSync(file) - database.exec('PRAGMA user_version = 54') + database.exec('PRAGMA user_version = 55') database.close() - expect(() => Database.open(file)).toThrow('SQLite schema version 54 is newer than the supported version 53.') + expect(() => Database.open(file)).toThrow('SQLite schema version 55 is newer than the supported version 54.') const reopened = new DatabaseSync(file) - expect(version(reopened)).toBe(54) + expect(version(reopened)).toBe(55) reopened.close() }) @@ -431,7 +431,7 @@ it('preserves old checkpoint bytes for explicit recovery validation', async () = const reopened = new DatabaseSync(file) try { - expect(version(reopened)).toBe(53) + expect(version(reopened)).toBe(54) expect(reopened.prepare('SELECT * FROM run_checkpoints').get()).toEqual({ run_id: 'run-a', checkpoint_json: '{"value":42}', @@ -464,7 +464,7 @@ it('upgrades version 14 while preserving existing Integration progress, subscrip Database.open(file).close() const upgraded = new DatabaseSync(file) try { - expect(version(upgraded)).toBe(53) + expect(version(upgraded)).toBe(54) const after = tables.map((table) => upgraded.prepare('SELECT * FROM ' + table).all()) expect(after.slice(0, 2)).toEqual( before.slice(0, 2).map((rows) => rows.map((row) => Object.fromEntries(Object.entries(row).filter(([key]) => key != 'operator_state')))), @@ -482,7 +482,7 @@ it('upgrades version 14 while preserving existing Integration progress, subscrip } }) -it('removes the event source tenant binding while preserving source configuration and queued events', async () => { +it('rejects legacy Source upgrades without modifying configuration or queued events', async () => { const file = await databaseFile() const database = legacyDatabase(file, 23) database @@ -496,31 +496,12 @@ it('removes the event source tenant binding while preserving source configuratio const events = database.prepare('SELECT * FROM source_events').all() database.close() - Database.open(file).close() - - const upgraded = new DatabaseSync(file) - try { - expect(upgraded.prepare('SELECT * FROM event_sources').get()).toEqual({ - source_id: source.source_id, - revision: source.revision, - name: source.name, - kind: 'feishu', - provider: source.provider, - external_identity: source.app_id, - connection_id: source.connection_id, - team_id: source.team_id, - event_types_json: source.event_types_json, - config_json: JSON.stringify({ manageSubscriptions: false }), - secrets_json: JSON.stringify({ verificationToken: source.verification_token, encryptKey: source.encrypt_key }), - state_json: JSON.stringify({ verifiedAt: source.verified_at }), - enabled: source.enabled, - last_received_at: source.last_received_at, - updated_at: source.updated_at, - }) - expect(upgraded.prepare('SELECT * FROM source_events').all()).toEqual(events) - } finally { - upgraded.close() - } + expect(() => Database.open(file)).toThrow('legacy_sources_require_explicit_migration') + const unchanged = new DatabaseSync(file) + expect(version(unchanged)).toBe(23) + expect(unchanged.prepare('SELECT * FROM event_sources').get()).toEqual(source) + expect(unchanged.prepare('SELECT * FROM source_events').all()).toEqual(events) + unchanged.close() }) it('adds Cron health without changing existing schedules', async () => { @@ -672,7 +653,7 @@ it('adds personal token storage to version 37 without changing existing accounts old.close() const upgraded = Database.open(file) try { - expect(version(upgraded.connection)).toBe(53) + expect(version(upgraded.connection)).toBe(54) expect(upgraded.connection.prepare('SELECT * FROM users').all()).toEqual(before) expect(upgraded.connection.prepare('SELECT * FROM user_tokens').all()).toEqual([]) expect(upgraded.connection.prepare('PRAGMA integrity_check').get()).toEqual({ integrity_check: 'ok' }) @@ -714,7 +695,7 @@ it('preserves existing personal tokens while adding Operator credential binding' const upgraded = Database.open(file) try { expect(upgraded.connection.prepare('SELECT * FROM user_tokens').get()).toEqual({ ...before, operator_fingerprint: null }) - expect(version(upgraded.connection)).toBe(53) + expect(version(upgraded.connection)).toBe(54) expect(upgraded.connection.prepare('PRAGMA integrity_check').get()).toEqual({ integrity_check: 'ok' }) } finally { upgraded.close() @@ -739,7 +720,7 @@ it('migrates shared Variables to the Operator without exposing or copying them t } }) -it('migrates source definitions without changing queued deliveries, deduplication or subscription ownership', async () => { +it('rejects legacy managed Sources while preserving deliveries, deduplication and subscription ownership', async () => { const file = await databaseFile() const old = legacyDatabase(file, 42) for (const [sourceId, verifiedAt] of [ @@ -765,23 +746,11 @@ it('migrates source definitions without changing queued deliveries, deduplicatio const before = tables.map((table) => old.prepare(`SELECT * FROM ${table}`).all()) old.close() - const upgraded = Database.open(file) - try { - expect(tables.map((table) => upgraded.connection.prepare(`SELECT * FROM ${table}`).all())).toEqual(before) - const sources = upgraded.connection.prepare('SELECT * FROM event_sources ORDER BY source_id').all() - expect(sources.map((source) => JSON.parse(String(source.state_json)))).toEqual([{}, { verifiedAt: 100 }]) - for (const source of sources) { - expect(source).toMatchObject({ kind: 'feishu', revision: 3, enabled: 0, team_id: 'team', last_received_at: 200, updated_at: 300 }) - expect(source.external_identity).toBe(`cli_${source.source_id}`) - expect(JSON.parse(String(source.config_json))).toEqual({ manageSubscriptions: true }) - expect(JSON.parse(String(source.secrets_json))).toEqual({ verificationToken: 'token', encryptKey: 'key' }) - } - } finally { - upgraded.close() - } - const reopened = Database.open(file) - expect(tables.map((table) => reopened.connection.prepare(`SELECT * FROM ${table}`).all())).toEqual(before) - reopened.close() + expect(() => Database.open(file)).toThrow('legacy_sources_require_explicit_migration') + const unchanged = new DatabaseSync(file) + expect(tables.map((table) => unchanged.prepare('SELECT * FROM ' + table).all())).toEqual(before) + expect(unchanged.prepare('SELECT COUNT(*) AS count FROM event_sources').get()).toEqual({ count: 2 }) + unchanged.close() }) it.each(['https://connector.oomol.com/', 'https://connector.example.com/'])( @@ -832,3 +801,29 @@ it('preserves legacy team bindings and remembers names across database reopen', reopened.close() } }) + +it('refuses legacy Slack snapshots without modifying their data or schema version', async () => { + const file = await databaseFile() + const old = legacyDatabase(file, 53) + old + .prepare(`INSERT INTO event_sources (source_id, revision, name, kind, provider, external_identity, connection_id, team_id, + config_json, secrets_json, state_json, event_types_json, updated_at) + VALUES ('legacy', 1, 'Slack', 'slack', 'slackbot', 'identity', 'connection', NULL, '{"appId":"A123"}', '{}', '{}', '["app_mention"]', 1)`) + .run() + old.prepare("INSERT INTO event_source_apps VALUES ('slack', 'A123', 'legacy-secret', NULL)").run() + old.close() + expect(() => Database.open(file)).toThrow('legacy_sources_require_explicit_migration') + const unchanged = new DatabaseSync(file) + try { + expect(version(unchanged)).toBe(53) + expect(unchanged.prepare('SELECT signing_secret FROM event_source_apps').get()).toEqual({ signing_secret: 'legacy-secret' }) + expect( + unchanged + .prepare('PRAGMA table_info(event_sources)') + .all() + .some((column) => column.name === 'connector_json'), + ).toBe(false) + } finally { + unchanged.close() + } +}) diff --git a/apps/server/test/poll.test.ts b/apps/server/test/poll.test.ts index 11eaa049..0ecde789 100644 --- a/apps/server/test/poll.test.ts +++ b/apps/server/test/poll.test.ts @@ -734,3 +734,36 @@ it.each([false, true])('admits Provider-defined Poll outputs after deduplication await closeService(service) } }) + +it('admits a complete event batch as one Run with capacity for only one pending Run', async () => { + const file = await databaseFile() + const definition: PollDefinition = { + snapshot, + buildOutputs: eventsPollOutputs, + poll: async ({ checkpoint }) => ({ + checkpoint: Number(checkpoint) + 1, + events: checkpoint == null ? [] : ['one', 'two', 'two', 'three'].map((id) => ({ dedupeKey: id, payload: { id } })), + }), + } + const service = await openService(file, { + capabilities: { connector: () => connector }, + clock: () => publishedAt, + runtime: { maxPendingRuns: 1 }, + triggerDefinitions: [definition], + }) + const database = new DatabaseSync(file) + try { + await publish(service, revision()) + await service.tickListeners('2026-08-21T00:01:00.000Z') + await service.tickListeners('2026-08-21T00:02:00.000Z') + const rows = database.prepare('SELECT trigger_outputs FROM runs').all() as { trigger_outputs: string }[] + expect(rows.map((row) => JSON.parse(row.trigger_outputs))).toEqual([{ events: [{ id: 'one' }, { id: 'two' }, { id: 'three' }] }]) + expect(service.pollState('main', 'poll')).toMatchObject({ checkpoint: 2, health: 'healthy' }) + await service.tickListeners('2026-08-21T00:03:00.000Z') + expect(database.prepare('SELECT COUNT(*) AS count FROM runs').get()).toEqual({ count: 1 }) + expect(database.prepare('SELECT COUNT(*) AS count FROM poll_event_dedupe').get()).toEqual({ count: 3 }) + } finally { + database.close() + await closeService(service) + } +}) diff --git a/apps/server/test/runAudit.test.ts b/apps/server/test/runAudit.test.ts index d1fb4e41..8f239049 100644 --- a/apps/server/test/runAudit.test.ts +++ b/apps/server/test/runAudit.test.ts @@ -78,7 +78,7 @@ it('retains audit after restart and migrates old Runs without inventing identiti database.exec('ALTER TABLE runs DROP COLUMN initiator_actor_id') database.exec('ALTER TABLE runs DROP COLUMN initiator_channel') database.exec( - 'DROP INDEX wait_auto_continue_due; ALTER TABLE wait_receipts DROP COLUMN auto_continue_at; ALTER TABLE wait_receipts DROP COLUMN resolution_source; DROP TABLE webhook_responses; DROP TABLE event_source_apps; DROP TABLE source_event_versions; PRAGMA user_version = 49', + 'DROP INDEX wait_auto_continue_due; ALTER TABLE wait_receipts DROP COLUMN auto_continue_at; ALTER TABLE wait_receipts DROP COLUMN resolution_source; DROP TABLE webhook_responses; DROP TABLE event_source_apps; DROP TABLE source_event_versions; DROP TABLE event_source_namespace; ALTER TABLE event_sources DROP COLUMN connector_json; ALTER TABLE event_sources DROP COLUMN pending_json; PRAGMA user_version = 49', ) database.close() const migrated = await openService(file) diff --git a/apps/server/test/sharedEventSources.test.ts b/apps/server/test/sharedEventSources.test.ts index 97045470..2714230c 100644 --- a/apps/server/test/sharedEventSources.test.ts +++ b/apps/server/test/sharedEventSources.test.ts @@ -7,7 +7,7 @@ import { expect, it } from 'vitest' import { createServerApp } from '../node/transport/http.ts' import { createConnectorHost } from './connectorHost.ts' import { openService } from './serviceFixture.ts' -import { sourceProtocolFixture } from './sourceProtocolFixture.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' const now = 1_800_000_000_000 function delivery(payload: unknown) { @@ -33,14 +33,15 @@ const event = (id: string) => ({ }, }) it('shares an app ingress while isolating workspaces, retaining fanout, and keeping the other source alive when one is disabled', async () => { + const respond = sourceReferenceFixture() const connector = createConnectorHost({ trigger: async (_provider, connectionId, _trigger, request) => { - if (request.operation !== 'source') throw new Error('Unexpected operation') + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') if (request.request.action === 'receive') { const supplied = JSON.parse(Buffer.from(request.request.rawBody, 'base64').toString()) as { recipient?: string } if (supplied.recipient != null && supplied.recipient !== connectionId) return { response: { status: 200, body: '{}', headers: {} } } } - return sourceProtocolFixture(request.request, connectionId) + return respond(request.request) }, listConnections: async () => ['1', '2'].map((id) => ({ @@ -114,6 +115,26 @@ it('shares an app ingress while isolating workspaces, retaining fanout, and keep flows.push(flowId) } + const protectedSource = sources[0]! + expect( + ( + await app.request(`/v1/event-sources/${protectedSource.sourceId}`, { + method: 'DELETE', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ version: 1, expectedRevision: protectedSource.revision }), + }) + ).status, + ).toBe(409) + expect( + ( + await app.request(`/v1/event-sources/${protectedSource.sourceId}`, { + method: 'PUT', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ version: 1, expectedRevision: protectedSource.revision, name: protectedSource.name, enabled: true, eventTypes: ['message.im'] }), + }) + ).status, + ).toBe(409) + for (let retry = 0; retry < 2; retry++) expect((await app.request(endpoint, delivery(event('1')))).status).toBe(200) await service.tickIntegration() expect(flows.map((flow) => service.control.runs.listRuns('operator', flow, 20).page.runs.length)).toEqual([1, 1, 0]) @@ -129,3 +150,69 @@ it('shares an app ingress while isolating workspaces, retaining fanout, and keep expect(flows.map((flow) => service.control.runs.listRuns('operator', flow, 20).page.runs.length)).toEqual([1, 1, 1]) expect(await (await app.request('/v1/event-sources')).text()).not.toContain('"signingSecret"') }) + +it('does not publish a new consumer while Source deletion is waiting for Connector', async () => { + const respond = sourceReferenceFixture() + const deleting = Promise.withResolvers() + const release = Promise.withResolvers() + const connector = createConnectorHost({ + listConnections: async () => [{ connectionId: 'connection', serviceId: 'slackbot', displayName: 'Slack', status: 'active', isDefault: true }], + trigger: async (_provider, _connection, _trigger, request) => { + if (request.operation !== 'sourceReference') throw new Error('Unexpected operation') + if (request.request.action === 'delete') { + deleting.resolve() + await release.promise + } + return respond(request.request) + }, + }) + const service = await openService(':memory:', { + capabilities: { connector: () => connector, integration: () => ({ callbackKey: 'key', publicOrigin: 'https://flow.example' }) }, + }) + const app = createServerApp(service, { resolveControlActor: () => 'operator' }) + const source = await service.control.eventSources.createEventSource( + 'operator', + { + kind: 'slack', + version: 1, + name: 'Slack', + connectionId: 'connection', + teamId: null, + appId: 'A123', + workspaceId: 'T123', + signingSecret: 'secret', + eventTypes: ['app_mention'], + }, + new AbortController().signal, + ) + expect( + (await app.request(new URL(source.endpointUrl!).pathname, delivery({ verify: true, response: { status: 200, body: 'verified', headers: {} } }))).status, + ).toBe(200) + const removed = service.control.eventSources.deleteEventSource('operator', source.sourceId, source.revision) + await deleting.promise + try { + const created = await service.control.flows.createFlow('operator', 'Concurrent publication', 'create-concurrent') + const flowId = created.flow.flowId + const draft = await service.control.drafts.changeDraft('operator', flowId, created.flow.draftRevisionId, [ + { + kind: 'graph.node.create', + nodeId: 'slack', + node: { + kind: 'integration', + name: 'Slack', + connectionId: 'connection', + definition: integrationDefinitions.find((item) => item.snapshot.key === 'slackbot.on_event')!.snapshot, + config: inputValues({ sourceId: source.sourceId, eventTypes: ['app_mention'] }), + }, + }, + ]) + await expect( + service.control.publications.publishFlow('operator', flowId, draft.revision.revisionId, currentEngineContract, null, 'publish-concurrent'), + ).rejects.toThrow('cannot be changed safely') + expect((await service.control.eventSources.listEventSources('operator')).sources[0]!.consumers).toEqual([]) + } finally { + release.resolve() + await removed + } + expect((await service.control.eventSources.listEventSources('operator')).sources).toEqual([]) +}) diff --git a/apps/server/test/sourceProtocolFixture.ts b/apps/server/test/sourceProtocolFixture.ts index 6f655042..e8825112 100644 --- a/apps/server/test/sourceProtocolFixture.ts +++ b/apps/server/test/sourceProtocolFixture.ts @@ -1,70 +1,95 @@ -import type { SourceConfiguration, SourceOperation } from '@oomol-lab/open-flow/event-source' import type { JsonValue } from '@oomol-lab/open-flow/flow-change' import { ConnectorTaskError } from '../node/deployment/connector-contract.ts' -const projection = (source: SourceConfiguration): SourceConfiguration => ({ - ...source, - state: { - ...source.state, - projection: { - ready: typeof source.state.verifiedAt === 'number', - view: { - appId: String(source.config.appId), - manageSubscriptions: true, - verificationTokenConfigured: true, - encryptKeyConfigured: true, - verifiedAt: typeof source.state.verifiedAt === 'number' ? new Date(source.state.verifiedAt).toISOString() : null, - }, - }, - }, -}) - -export function sourceProtocolFixture(request: SourceOperation, accountId = 'cli_test', now = 1_800_000_000_000): unknown { - switch (request.action) { - case 'initialize': { - if (!accountId) throw new ConnectorTaskError('connector.input-invalid', 'Connector did not confirm the application identity.') - const input = request.input +export function sourceReferenceFixture() { + const sources = new Map() + let revision = 0 + return (request: import('@oomol-lab/open-flow/event-source').SourceReferenceOperation, accountId = 'cli_test'): unknown => { + if (request.action === 'route') { + const body = JSON.parse(Buffer.from(request.rawBody, 'base64').toString()) as { verify?: boolean; response?: unknown } + if (body.verify) + for (const source of sources.values()) + if (source.reference.applicationId === request.applicationId) { + source.ready = true + source.view.verifiedAt = new Date(1_800_000_000_000).toISOString() + source.revision = ++revision + } return { - source: projection({ - identity: JSON.stringify([input.teamId, input.connectionId, accountId]), - eventTypes: input.eventTypes as string[], - config: { appId: input.appId ?? accountId, accountId }, - secrets: { verificationToken: input.verificationToken ?? '', encryptKey: input.encryptKey ?? '', signingSecret: input.signingSecret ?? '' }, - state: {}, - }), + references: body.response + ? [] + : [...sources.values()].filter((source) => source.reference.applicationId === request.applicationId).map((source) => source.reference), + response: body.response, } } - case 'update': { - const secrets = { ...request.source.secrets } - let rotated = false - for (const field of ['verificationToken', 'encryptKey', 'signingSecret']) - if (request.input[field] != null && request.input[field] !== secrets[field]) { - secrets[field] = request.input[field]! - rotated = true - } - return { - source: projection({ ...request.source, eventTypes: request.input.eventTypes as string[], secrets, state: rotated ? {} : request.source.state }), + if (request.action === 'create') { + if (!accountId) throw new ConnectorTaskError('connector.input-invalid', 'Connector did not confirm the application identity.') + const existing = sources.get(request.requestKey) + if (existing != null) return { source: existing } + const source = { + reference: { applicationId: String(request.input.appId ?? accountId), sourceId: request.requestKey }, + revision: ++revision, + eventTypes: [...(request.input.eventTypes as string[])], + ready: false, + view: { + appId: String(request.input.appId ?? accountId), + workspaceId: request.input.workspaceId as string | undefined, + projectId: request.input.project != null ? '100' : undefined, + signingSecretConfigured: true, + manageSubscriptions: Boolean(request.input.manageSubscriptions), + verificationTokenConfigured: request.input.verificationToken != null, + encryptKeyConfigured: request.input.encryptKey != null, + verifiedAt: null, + }, } + sources.set(request.requestKey, source) + return { source } } - case 'inspect': - return { source: projection(request.source) } - case 'prepare': - return { - source: projection(request.source), - subscriptions: request.config.resource == null ? [] : [{ key: JSON.stringify(request.config.resource), config: request.config }], - } - case 'receive': { - // The test supplies Connector responses as the callback body; no provider protocol runs here. - const result = JSON.parse(Buffer.from(request.rawBody, 'base64').toString()) as { - response?: unknown - event?: Record - verify?: boolean + const sourceId = 'requestKey' in request.reference ? request.reference.requestKey : request.reference.sourceId + const source = sources.get(sourceId) + if (source == null && request.action === 'delete') return { deleted: true } + if (source == null) throw new ConnectorTaskError('connector.input-invalid', 'Source not found.') + switch (request.action) { + case 'get': + return { source } + case 'delete': + sources.delete(sourceId) + return { deleted: true } + case 'update': { + source.eventTypes = [...(request.input.eventTypes as string[])] + source.revision = ++revision + if (request.input.signingSecret != null || request.input.encryptKey != null || request.input.verificationToken != null) + for (const item of sources.values()) { + item.ready = false + item.view = { ...item.view, verifiedAt: null } + item.revision = revision + } + return { source } } - return { - source: projection({ ...request.source, state: result.verify ? { ...request.source.state, verifiedAt: now } : request.source.state }), - ...result, - event: result.event == null ? undefined : { ...result.event, matches: request.consumers.map((consumer) => consumer.id) }, + case 'resource': + source.ready = request.active + source.view = { + ...source.view, + subscriptionStatus: request.active ? 'ready' : 'idle', + verifiedAt: request.active ? new Date(1_800_000_000_000).toISOString() : null, + } + return { source, outcome: 'ready' } + case 'prepare': + if (!source.ready && !source.view.projectId) throw new ConnectorTaskError('connector.input-invalid', 'Verify the Source first.') + return { source, subscriptions: request.config.resource == null ? [] : [{ key: JSON.stringify(request.config.resource) }] } + case 'receive': { + const result = JSON.parse(Buffer.from(request.rawBody, 'base64').toString()) as { + verify?: boolean + event?: Record + response?: unknown + } + if (result.verify) + for (const item of sources.values()) { + item.ready = true + item.view = { ...item.view, verifiedAt: new Date(1_800_000_000_000).toISOString() } + item.revision = ++revision + } + return { source, ...result, event: result.event == null ? undefined : { ...result.event, matches: request.consumers.map((consumer) => consumer.id) } } } } } diff --git a/apps/server/test/sourceReferences.test.ts b/apps/server/test/sourceReferences.test.ts new file mode 100644 index 00000000..ee40180b --- /dev/null +++ b/apps/server/test/sourceReferences.test.ts @@ -0,0 +1,310 @@ +import type { CreateEventSource, UpdateEventSource } from '@oomol-lab/open-flow/control-api' +import type { SourceReferenceOperation } from '@oomol-lab/open-flow/event-source' + +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { expect, it, onTestFinished } from 'vitest' +import { runEnvironment } from '../node/application/run-environment.ts' +import { ImplicitConnectorAccessHost } from '../node/deployment/connector-access.ts' +import { ConnectorTaskError } from '../node/deployment/connector-contract.ts' +import { EventSourceRuntime } from '../node/runtime/event-source-runtime.ts' +import { isolatedVmEngineDigest } from '../node/runtime/isolated-vm.ts' +import { Database } from '../node/storage/database.ts' +import { Store } from '../node/storage/store.ts' +import { createConnectorHost } from './connectorHost.ts' +import { sourceReferenceFixture } from './sourceProtocolFixture.ts' + +const input: Extract = { + kind: 'slack', + version: 1, + name: 'Slack', + connectionId: 'connection', + teamId: null, + appId: 'A123', + workspaceId: 'T123', + signingSecret: 'source-private-key', + eventTypes: ['app_mention'], +} +const signal = () => new AbortController().signal +function fixture() { + const directory = mkdtempSync(join(tmpdir(), 'source-reference-')) + const file = join(directory, 'runtime.db') + let database = Database.open(file) + let store = new Store(database, runEnvironment(isolatedVmEngineDigest), Date.now) + const respond = sourceReferenceFixture() + const requests: SourceReferenceOperation[] = [] + let fail: SourceReferenceOperation['action'] | undefined + let rejectEdit = false + let disconnected = false + let gate: { action: SourceReferenceOperation['action']; entered: () => void; wait: Promise; reject: boolean } | undefined + let receiveGate: Promise | undefined + const connector = createConnectorHost({ + listConnections: async () => + disconnected + ? [] + : [{ connectionId: 'connection', providerAccountId: 'U123', displayName: 'Slack', serviceId: 'slackbot', status: 'active', isDefault: true }], + trigger: async (_provider, _connection, _trigger, request) => { + if (request.operation !== 'sourceReference') throw new Error('Slack must only use references') + requests.push(structuredClone(request.request)) + if (disconnected && request.request.action !== 'delete') throw new ConnectorTaskError('connector.unavailable', 'Connection removed') + const delayed = gate?.action === request.request.action ? gate : undefined + if (delayed) gate = undefined + if (request.request.action === 'update' && rejectEdit) { + rejectEdit = false + throw new ConnectorTaskError('connector.input-invalid', 'The Source changed.', 409) + } + const result = structuredClone(respond(request.request)) + if (delayed) { + delayed.entered() + await delayed.wait + if (delayed.reject) throw new ConnectorTaskError('connector.input-invalid', 'Concurrent create conflict', 409) + } + if (request.request.action === 'receive' && receiveGate != null) await receiveGate + if (request.request.action === fail) { + fail = undefined + throw new ConnectorTaskError('connector.unavailable', 'Response lost') + } + return result + }, + }) + const runtime = () => + new EventSourceRuntime( + store, + () => connector, + new ImplicitConnectorAccessHost(), + () => ({ callbackKey: 'key', publicOrigin: 'https://flow.example' }), + Date.now, + () => {}, + ) + onTestFinished(() => { + database.close() + rmSync(directory, { recursive: true, force: true }) + }) + return { + runtime, + requests, + get store() { + return store + }, + get database() { + return database + }, + disconnect() { + disconnected = true + }, + delay(action: SourceReferenceOperation['action'], reject = false) { + const entered = Promise.withResolvers() + const release = Promise.withResolvers() + gate = { action, entered: entered.resolve, wait: release.promise, reject } + return { entered: entered.promise, release: release.resolve } + }, + rejectEdit() { + rejectEdit = true + }, + loseResponse(action: SourceReferenceOperation['action']) { + fail = action + }, + delayReceive(wait: Promise) { + receiveGate = wait + }, + reopen() { + database.close() + database = Database.open(file) + store = new Store(database, runEnvironment(isolatedVmEngineDigest), Date.now) + }, + } +} +const challenge = () => + new Request('https://flow.example/events', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ verify: true, response: { status: 200, body: 'verified', headers: {} } }), + }) + +it('recovers creation, editing and deletion after lost responses and restart without storing Slack secrets', async () => { + const test = fixture() + test.loseResponse('create') + await expect(test.runtime().create(input, signal())).rejects.toThrow('Response lost') + const reserved = test.store.eventSources.list('https://flow.example')[0]! + expect(reserved.lastError).toContain('unfinished') + expect(test.store.eventSources.ready(reserved.sourceId, 'binding')).toBe(false) + test.reopen() + const source = await test.runtime().create(input, signal()) + expect(source.sourceId).toBe(reserved.sourceId) + const creates = test.requests.filter((request) => request.action === 'create') + expect(creates[0]!.requestKey).toBe(creates[1]!.requestKey) + expect(creates[0]!.deploymentId).toBe(creates[1]!.deploymentId) + expect((await test.runtime().receive(source.sourceId, challenge())).status).toBe(200) + expect((await test.runtime().list()).sources[0]!.verifiedAt).not.toBeNull() + const edit: UpdateEventSource = { + version: 1, + expectedRevision: source.revision, + name: 'Renamed', + enabled: true, + eventTypes: ['message.im'], + signingSecret: 'replacement-private-key', + } + test.loseResponse('update') + await expect(test.runtime().update(source.sourceId, edit, signal())).rejects.toThrow('Response lost') + expect((await test.runtime().receive(source.sourceId, challenge())).status).toBe(503) + await expect(test.runtime().update(source.sourceId, { ...edit, name: 'Other' }, signal())).rejects.toThrow('unfinished') + test.reopen() + const updated = await test.runtime().update(source.sourceId, edit, signal()) + expect(updated).toMatchObject({ name: 'Renamed', eventTypes: ['message.im'], verifiedAt: null, revision: source.revision + 1 }) + expect((await test.runtime().receive(source.sourceId, challenge())).status).toBe(200) + const stored = test.database.connection.prepare('SELECT * FROM event_sources').all() + expect(JSON.stringify(stored)).not.toContain(input.signingSecret) + expect(JSON.stringify(stored)).not.toContain(edit.signingSecret!) + expect(test.database.connection.prepare('SELECT * FROM event_source_apps').all()).toEqual([]) + expect(test.store.eventSources.require(source.sourceId).secrets).toEqual({}) + expect(test.store.eventSources.require(source.sourceId).state).not.toHaveProperty('verifiedAt') + expect( + test.requests.filter((request) => !['create', 'update'].includes(request.action)).every((request) => !JSON.stringify(request).includes('signingSecret')), + ).toBe(true) + test.loseResponse('delete') + await expect(test.runtime().delete(source.sourceId, updated.revision, signal())).rejects.toThrow('Response lost') + expect(test.store.eventSources.require(source.sourceId).pending?.action).toBe('delete') + test.reopen() + await test.runtime().delete(source.sourceId, updated.revision, signal()) + expect((await test.runtime().list()).sources).toEqual([]) +}) + +it('keeps names and enabled state local and rejects an old callback after a shared secret edit', async () => { + const test = fixture() + const a = await test.runtime().create(input, signal()) + const b = await test.runtime().create({ ...input, workspaceId: 'T456', name: 'Second' }, signal()) + const before = test.requests.length + const local = await test + .runtime() + .update(a.sourceId, { version: 1, expectedRevision: a.revision, name: 'Local', enabled: false, eventTypes: a.eventTypes }, signal()) + expect(test.requests).toHaveLength(before) + expect(local).toMatchObject({ name: 'Local', enabled: false }) + const gate = Promise.withResolvers() + test.delayReceive(gate.promise) + const receive = test.runtime().receive(b.sourceId, challenge()) + await Promise.resolve() + const edited = await test + .runtime() + .update( + a.sourceId, + { version: 1, expectedRevision: local.revision, name: 'Local', enabled: true, eventTypes: a.eventTypes, signingSecret: 'replacement-private-key' }, + signal(), + ) + expect(edited.verifiedAt).toBeNull() + gate.resolve() + expect((await receive).status).toBe(409) + expect(test.store.eventSources.ready(b.sourceId, 'binding')).toBe(false) + expect((await test.runtime().list()).sources.every((source) => source.verifiedAt == null)).toBe(true) +}) + +it('releases a rejected edit without changing the live configuration or leaving the application blocked', async () => { + const test = fixture() + const source = await test.runtime().create(input, signal()) + expect((await test.runtime().receive(source.sourceId, challenge())).status).toBe(200) + test.rejectEdit() + await expect( + test.runtime().update( + source.sourceId, + { + version: 1, + expectedRevision: source.revision, + name: 'Rejected', + enabled: true, + eventTypes: ['message.im'], + }, + signal(), + ), + ).rejects.toMatchObject({ status: 409 }) + const stored = test.store.eventSources.require(source.sourceId) + expect(stored).toMatchObject({ name: source.name, eventTypes: source.eventTypes, pending: null, revision: source.revision }) + expect(test.store.eventSources.ready(source.sourceId, 'binding')).toBe(true) +}) + +it.each(['create', 'update'] as const)('deletes an unfinished %s after the connection disappears and the server restarts', async (action) => { + const test = fixture() + if (action === 'create') { + test.loseResponse('create') + await expect(test.runtime().create(input, signal())).rejects.toThrow('Response lost') + } else { + const source = await test.runtime().create(input, signal()) + test.loseResponse('update') + await expect( + test.runtime().update( + source.sourceId, + { + version: 1, + expectedRevision: source.revision, + name: 'Edited', + enabled: true, + eventTypes: ['message.im'], + }, + signal(), + ), + ).rejects.toThrow('Response lost') + } + test.disconnect() + test.reopen() + const source = test.store.eventSources.list('https://flow.example')[0]! + test.loseResponse('delete') + await expect(test.runtime().delete(source.sourceId, source.revision, signal())).rejects.toThrow('Response lost') + test.reopen() + await test.runtime().delete(source.sourceId, source.revision, signal()) + expect(test.store.eventSources.list('https://flow.example')).toEqual([]) +}) + +it.each(['create', 'update'] as const)('keeps deletion pending when an older %s response arrives', async (action) => { + const test = fixture() + const initial = action === 'update' ? await test.runtime().create(input, signal()) : undefined + const delayed = test.delay(action) + const operation = + initial == null + ? test.runtime().create(input, signal()) + : test.runtime().update( + initial.sourceId, + { + version: 1, + expectedRevision: initial.revision, + name: 'Edited', + enabled: true, + eventTypes: ['message.im'], + }, + signal(), + ) + const rejected = expect(operation).rejects.toThrow('unfinished') + await delayed.entered + const source = test.store.eventSources.list('https://flow.example')[0]! + test.loseResponse('delete') + await expect(test.runtime().delete(source.sourceId, source.revision, signal())).rejects.toThrow('Response lost') + delayed.release() + await rejected + expect(test.store.eventSources.require(source.sourceId).pending?.action).toBe('delete') + await test.runtime().delete(source.sourceId, source.revision, signal()) + expect(test.store.eventSources.list('https://flow.example')).toEqual([]) +}) + +it('does not remove a completed Source when a concurrent create returns a conflict', async () => { + const test = fixture() + const delayed = test.delay('create', true) + const first = test.runtime().create(input, signal()) + const rejected = expect(first).rejects.toMatchObject({ status: 409 }) + await delayed.entered + const created = await test.runtime().create(input, signal()) + delayed.release() + await rejected + expect(test.store.eventSources.require(created.sourceId)).toMatchObject({ pending: null, connector: { reference: expect.any(Object) } }) +}) + +it.each([false, true])('preserves application callback cancellation when the Connector response is lost: %s', async (lost) => { + const test = fixture() + await test.runtime().create(input, signal()) + const delayed = test.delay('route') + if (lost) test.loseResponse('route') + const controller = new AbortController() + const receiving = test.runtime().receiveApplication('slack', input.appId, new Request(challenge(), { signal: controller.signal })) + await delayed.entered + controller.abort() + delayed.release() + await expect(receiving).rejects.toMatchObject({ name: 'AbortError' }) + expect(test.requests.filter((request) => request.action === 'receive')).toEqual([]) +}) diff --git a/apps/server/test/workflowInterfaceMigration.test.ts b/apps/server/test/workflowInterfaceMigration.test.ts index af14b399..b577184c 100644 --- a/apps/server/test/workflowInterfaceMigration.test.ts +++ b/apps/server/test/workflowInterfaceMigration.test.ts @@ -81,7 +81,7 @@ async function fixture() { } function closeLegacy() { database.connection.exec( - "UPDATE publish_operations SET input_json = json_remove(input_json, '$.interface'); ALTER TABLE publications DROP COLUMN interface_json; ALTER TABLE runs DROP COLUMN initiator_actor_id; ALTER TABLE runs DROP COLUMN initiator_channel; DROP INDEX wait_auto_continue_due; ALTER TABLE wait_receipts DROP COLUMN auto_continue_at; ALTER TABLE wait_receipts DROP COLUMN resolution_source; DROP TABLE webhook_responses; DROP TABLE event_source_apps; DROP TABLE source_event_versions; PRAGMA user_version = 48;", + "UPDATE publish_operations SET input_json = json_remove(input_json, '$.interface'); ALTER TABLE publications DROP COLUMN interface_json; ALTER TABLE runs DROP COLUMN initiator_actor_id; ALTER TABLE runs DROP COLUMN initiator_channel; DROP INDEX wait_auto_continue_due; ALTER TABLE wait_receipts DROP COLUMN auto_continue_at; ALTER TABLE wait_receipts DROP COLUMN resolution_source; DROP TABLE webhook_responses; DROP TABLE event_source_apps; DROP TABLE source_event_versions; DROP TABLE event_source_namespace; ALTER TABLE event_sources DROP COLUMN connector_json; ALTER TABLE event_sources DROP COLUMN pending_json; PRAGMA user_version = 48;", ) database.close() } @@ -176,7 +176,7 @@ it('upgrades full and delta Revisions, preserves identities and Live history, an }) await closeService(service) const reopened = Database.open(file) - expect(reopened.connection.prepare('PRAGMA user_version').get()).toEqual({ user_version: 53 }) + expect(reopened.connection.prepare('PRAGMA user_version').get()).toEqual({ user_version: 54 }) reopened.close() }) diff --git a/docs/authoring/cli-lab.md b/docs/authoring/cli-lab.md index f0af6f6b..0e2777ef 100644 --- a/docs/authoring/cli-lab.md +++ b/docs/authoring/cli-lab.md @@ -371,3 +371,8 @@ Connector calls. Run both reference transports: bun run lab test wait-auto-continue bun run lab test wait-auto-continue --mcp ``` + +`poll-batch` verifies public Poll creation, read and local update, then executes two deterministic +Draft Runs with two Gmail events each. Run `bun run lab test poll-batch` and +`bun run lab test poll-batch --mcp`. Both events must reach Outputs together as a messages array. +Provider polling, page admission, deduplication and restart behavior are covered by Server Poll tests. diff --git a/docs/triggers/architecture-plan.md b/docs/triggers/architecture-plan.md new file mode 100644 index 00000000..17aa75b8 --- /dev/null +++ b/docs/triggers/architecture-plan.md @@ -0,0 +1,182 @@ +# Trigger 架构收敛实施计划 + +范围修正(2026-10-27):本轮将 Source 配置、秘密和远端资源生命周期交由 Connector 管理,保留 Poll 的批量输出和执行粒度。新建与已有 Poll 均使用 definitionVersion 2 和 `events[]`,无需升级。此前单事件方案撤回,Source 路由和回调改造继续保留。 + +状态:本轮代码调整与自动化验收已完成。2026-10-27 按用户要求一次覆盖全部现有 Source Provider,真实账号验收由用户手动执行;部署前的版本和存量复核仍待上线时完成,未部署。 + +2026-10-27 已只读核实目标 production 两个租户分片的 Source 及相关记录均为空。Server 迁移 54 对任何旧 Source 明确拒绝并保留原数据。发布前仍需复查部署版本和存量;本地测试不等于真实第三方验收。 + +实现与兼容边界见 [操作契约](connector-contract.md) 和 [发布基线](release-baseline.md)。以下阶段保留设计依据,当前实现不再限于首批 Slack。 + +保留现有 Publication、调度、去重、恢复和 Run 准入机制,收敛事件语义与 Connector 协议。目标是让新增 Provider 主要发生在 Connector,让同一 Trigger 在两个 Connector 后端具有一致的业务行为,并让用户不必因为采集方式改变而重写 Flow。 + +## 目标边界 + +| 所有者 | 持有的数据和行为 | +| ------------------------ | --------------------------------------------------------------------------------------------- | +| Open Flow 公共包 | 节点业务配置、输出契约、公共 authoring、协议消费与输出验证、部署 conformance | +| Open Flow Server | Publication、消费者绑定、调度、业务读取 checkpoint、事件去重、持久化投递、Run 准入与恢复 | +| Connector | Provider 配置和身份验证、签名秘密、上游请求、远端订阅与续租执行、Provider Source 状态及其投影 | +| connect 部署适配 | persistent token、连接和 verified account 的授权与订阅归属、存储和维护调度 | +| oomol-connector 部署适配 | Team、principal、app 的授权与订阅归属、存储和维护调度 | + +保留 Open Flow 的公开回调入口。Connector 负责验证第三方回调并返回规范化事件或握手响应,Open Flow 负责可靠保存和投递。此轮不把 Connector 改造成承载全部事件队列和业务调度的平台。 + +共享应用入口保留三个层次:application ingress 持有应用级 signing secret 和验证状态;账号或项目 Source 引用该入口并持有来源身份与配置;Flow consumer 持有业务过滤条件和消费关系。前两层由 Connector 持久化,消费者由 Open Flow 管理。没有应用共享入口的 Provider 保持自己的 Source 模型。 + +应用级状态不能按 Source 复制。阶段一明确 application 的授权和共享作用域;同一授权作用域内的多个 Source 引用同一份应用状态,不能仅凭相同的 `app_id` 合并不同 Team 或部署的资源。跨作用域共享需要明确授权契约,否则拒绝合并。 + +Open Flow 保存的 Source 引用和消费者配置不包含 Provider 验签秘密或远端资源 ID。Connector 的状态投影由 Connector 生成;Open Flow 自己的发布准备进度、投递积压和 Run 错误独立保存。 + +业务 checkpoint 继续归 Open Flow,必须与事件准入一致提交。Connector 的订阅恢复状态不能被当成业务消费进度,两者需要在协议中分别命名和解释。 + +## 保留的运行保证 + +- 所有候选资源准备成功后,才能原子切换 Live 并激活绑定;失败保留旧 Live。 +- 无法独立准备候选资源的修改明确拒绝,不能先拆除线上资源再补偿。 +- 当前 Publication、binding 和 runtime version 决定准入;旧 worker 和候选回调不能绕过检查。 +- 关闭 Flow 阻止新的生产准入,已接受的 Run 按原有机制继续执行。 +- 通知唤醒和扫描进度分别持久化;扫描期间到达的通知不能丢失。 +- 同一准入身份至多创建一个普通 Run。这个保证不等同于第三方事件完整性或外部副作用 exactly-once。 +- 删除最后一个消费者才释放共享资源;撤权、续租失败和清理失败保持可诊断、可恢复。 +- Source 修改和删除必须与发布、消费者变更保持一致:明确失败的操作不能改变当前 Live 使用的有效配置;已提交但响应丢失的操作按同一身份查询或重试,不能被当成未执行。跨服务调用不能替代本地原子约束。 +- 应用密钥轮换使所有关联 Source 的旧验证状态一起失效,有效 challenge 更新共同验证状态;删除一个 Source 不影响其他 Source 的应用入口,最后一个引用完成清理后才能删除应用状态。 + +## 阶段一 固定契约和发布基线 + +目的:在跨三个仓库修改之前,确定共同语义及真实兼容义务。 + +1. 核实三个项目实际发布和部署的版本,记录 artifact、源码提交、协议能力及受支持的旧版本。按 [contract evolution](../control/contracts/compatibility.md#release-baselines-and-contract-versioning) 判断版本边界;本计划不预先指定新版本号,也不按 PR 次数递增契约。 +2. 编写共同 Trigger 协议规范,明确 `read`、`options`、订阅控制、callback receive 与 Source 管理的请求和响应。协议覆盖事件身份、checkpoint、分页、错误分类、取消、能力缺失及不支持版本的拒绝行为。 +3. 定义去重作用域和保留期限:相同配置重新发布、改变过滤条件、切换连接、重新创建 Source、手动重放分别是否产生新准入身份。不能把各路径现有不同的哈希公式直接当作统一产品语义。 +4. 选择共同协议和 fixtures 的唯一维护位置。优先放在公开 connect 的独立、可版本化契约产物中,由另两个仓库消费;避免让托管服务依赖整个 Open Flow 产品包或 connect Server。 + +交付物:协议规范、版本兼容表、旧数据和旧 Flow 的迁移清单、三仓库的依赖与发布顺序。 + +验收:同一事件经不同分页、重复通知或不同后端处理,预期业务结果和准入身份可以被明确描述;旧版本不兼容时能在创建副作用前拒绝。 + +## 阶段二 建立双后端协议验收 + +先用测试固定现有有效行为,再调整状态所有权。共享 fixtures 和断言消费公开操作,不复制 Provider 的签名、事件解析或状态机实现。 + +| 场景 | 必须观察的结果 | +| ----------------------------------------- | ---------------------------------------------------- | +| 正常事件、重复回调、相同事件不同 delivery | 规范化输出和业务身份一致,重复不新增准入 | +| 初始化与分页续读、重启后继续 | checkpoint 含义一致,不丢页、不错误跳过数据 | +| 临时错误、权限失效、非法配置 | 错误分类一致,调用方能决定重试或重新授权 | +| 创建响应丢失、取消、重复清理 | 不重复创建远端资源,保留未完成清理记录 | +| 候选订阅握手与旧回调 | 握手可完成,候选不能启动生产 Run,旧绑定不能继续准入 | +| 版本或能力不匹配 | 明确拒绝,不回落到原始 proxy 或静默改变输出 | + +Provider 首批选用 Gmail Poll、GitHub Watch PR、Tally webhook、Asana 握手、Slack 共享应用回调和 Jira 共享托管资源,分别覆盖不同协议形态。其余 Provider 在受影响的共享路径通过后逐步接入。 + +connect 的 token ownership 与托管 Connector 的 Team/principal ownership 分别验证。权限差异不要求返回完全相同的结果,但必须满足各自的隔离、撤权和清理契约。 + +交付物:两个后端可运行的同一套协议 fixtures、目录兼容检查,以及 Open Flow 消费端的协议测试。 + +验收:故意引入输出、事件身份或错误分类差异时,测试能够在对应边界失败;不能只比较 schema 或静态目录是否相等。 + +## 阶段三 收敛 Source 状态所有权 + +主要改动位于两个 Connector 的 `sources`、订阅存储和管理接口,以及 Open Flow 的 `event-source-runtime`、`event-source-store` 和公共 Source 契约。 + +本阶段已从 Slack 的纵向闭环扩展至全部 7 个现有 Provider。下面规定行为和所有权,不预先要求通用配置版本、操作日志或后台清理框架。 + +1. Connector 持久化应用共享秘密、验证状态和 Source 配置,返回不透明引用。新建 Source 直接进入正式存储;读取、准备和回调按引用操作,不再往返传递整份 Source 和秘密。 +2. Open Flow 保存本地 Source ID、Connector 引用、消费者关系与部署状态,接通创建、读取、验证、回调、可靠投递和删除。Provider 投影由 Connector 生成,移除 Open Flow 对其内部字段的修改;保留现有发布、去重和准入检查。 +3. 保留现有编辑能力,按字段所有权处理:名称和启用状态由 Open Flow 管理,Provider 事件类型配置由 Connector 管理。修改事件类型时复核现有及准备中消费者的需求;与发布并发或明确失败的修改不能破坏当前有效配置。先检查实际调用链和现有事务能力,只为必须跨服务协调的操作增加必要状态,不预设不可变候选配置或通用版本系统。 +4. signing secret 在应用级修改,所有关联 Source 的验证状态一起失效,再通过有效 challenge 重新验证;旧在途结果不能恢复新秘密的 readiness。明确告知修改会影响该应用下的所有 Sources,不实现无中断轮换或双密钥过渡。 +5. 删除复用现有消费者和发布约束,远端删除保持幂等;不能在删除过程中新增有效消费者。远端调用失败或结果未知时保留足够的引用信息供重试,不能提前丢失清理对象。按实际失败路径决定必要的本地状态,不建设通用操作日志或操作结果查询服务。 +6. 读取、使用、修改和删除分别校验真实归属,连接换账号或撤权后不得继续处理事件。检查现有授权与撤权入口,优先直接接入;有权限的管理方在连接失效后仍应能删除本地持久化 Source,不能要求有效的上游凭据。Slack 当前没有需要注销的托管远端订阅,不以独立后台清理系统作为接入前置条件。其他 Provider 的续租、远端撤销和重试在接入该 Provider 时按实际资源生命周期处理。 +7. 上线前只读复查目标环境的 Source、订阅、准备记录、投递记录和消费者绑定仍为空。若出现真实存量,再决定处置方式;不提前实现导入、暂存、迁移日志或长期双写。当前核验不代表所有自托管和开发环境均无数据。 + +交付物:两个 Connector 与 Open Flow 的全部 Source 引用闭环、现有编辑和删除能力、共享秘密修改后的重新验证,以及对应协议与部署说明。响应丢失时明确如何通过现有读取或幂等重试确认状态,不要求额外的通用操作基础设施。 + +验收覆盖以下行为: + +- 从 Open Flow 创建 Source,读取引用和验证状态,接收 challenge 与事件并可靠投递;重启后可继续使用,回调重试不增加重复 Run。 +- 修改名称、启用状态和事件类型保持现有能力;移除消费者所需事件类型被拒绝,与发布并发或明确失败的修改不破坏当前有效配置。 +- 创建响应丢失后重试不重复创建;修改、删除结果未知时可确认当前状态或安全重试,删除期间不能新增有效消费者。 +- 同一应用两个 Sources 共用秘密和验证状态;修改秘密后共同失效,有效 challenge 恢复共同状态,旧在途结果不能恢复 readiness。 +- 删除一个 Source 不影响另一个,删除最后一个清空当前秘密和验证状态;连接失效后仍能由有权限的管理方删除。不同归属不能访问或修改彼此资源,撤权后不能继续处理事件。 +- 秘密不出现在普通读取、日志或节点输出中。 + +正式链路切换时同步更新消费者并移除不再使用的代码。只保留有实际发布义务的兼容行为;协议已发布不等于存在需要迁移的 Source 数据。 + +## 阶段四 移除 Open Flow 的 Provider 协议分支 + +1. 由 Connector 定义回调需要的 headers、query 和 body 能力;Open Flow 提供受大小限制的通用转发。回调中的授权头作为受保护输入,与调用 Connector 的认证头隔离。 +2. 解决创建期间的握手关联。Connector 必须在发起上游创建前持久化可用于握手的关联身份,Open Flow 能在正式订阅激活前调用验证路径。 +3. 验证 Asana 创建请求等待 callback 的场景不会与 Connector 订阅锁互相等待。握手处理不能依赖尚未完成的创建结果,也不能借机激活生产绑定。 +4. 删除 `remote.ts` 中的 Asana 名称分支与手写 Provider header 清单。Provider 解析与握手响应归 Connector;Open Flow 只执行通用协议。 + +交付物:通用回调适配、Connector 握手实现和双后端行为测试。 + +验收:增加一个具有新签名 header 的普通 webhook Provider,不需要增加 Open Flow 的 Provider 名称分支;创建响应丢失和握手重试仍可恢复。 + +## 阶段五 保持 Poll 批量语义 + +Source 所有权调整不改变 Poll 执行粒度。保持 definitionVersion 2 和 `events[]`,每个非空去重结果页作为一个批次准入 Run。首轮 baseline、过滤、重叠去重、续页、重启恢复和 runtime fencing 保持原行为。 + +删除单事件 delivery 分支、旧定义快照目录和 Trigger refresh 升级路径,不增加模式开关。CLI/MCP Lab `poll-batch` 验证多条消息一起进入一个 Run;Server 测试覆盖去重和容量不足时 checkpoint 不推进。 + +## 阶段六 优化共享应用回调路由 + +本阶段只优化路由和性能。阶段三已保证共享秘密和验证状态一致,以及修改秘密后的重新验证;这些保证不依赖路由优化,不要求无中断轮换或独立后台清理系统。 + +1. 建立同一应用下 1、10、100 个 Source 的测试,记录每次回调的 Connector 调用数、响应延迟和投递结果。加入一个来源连接失效、慢响应与重复回调。 +2. Connector 在应用级验证签名并返回规范化路由身份,Open Flow 按已验证身份选出匹配 Source,再进行必要的来源验证与业务处理。未验证的 payload 只能用于候选查找,不能授权投递。 +3. 保留来源和消费者版本的最终复核,避免签名有效但 workspace/account 不匹配的事件进入其他 Flow。 +4. 对匹配目标持久化投递后再确认普通事件;握手和挑战按各自协议同步响应。一个无关 Source 的失效不应阻断目标 Source。 + +交付物:通用应用路由协议、受影响 Provider 适配和前后对比结果。 + +验收:只匹配一个 Source 的事件不会随应用全部 Source 数量线性增加 Connector 调用;无关来源故障不影响目标投递;真正多目标事件仍对每个目标正确去重。若当前规模尚不构成问题,可在完成基线测量后延期本阶段。 + +## 实施顺序与发布 + +依赖关系:阶段一 → 阶段二 → 阶段三 → 阶段四。阶段五依赖阶段一、二的契约和测试,可作为独立发布批次;阶段六依赖稳定的 Source 身份与回调协议。 + +每阶段保持可独立审查和验证,不把三个仓库的全部改造压成一次发布。先交付一个纵向闭环,再扩展 Provider 数量。协议测试可以先落地,破坏性行为必须与实际兼容义务对应的升级或拒绝机制一起交付。 + +新 Source 创建后,回滚必须考虑新存储中的配置、秘密和远端资源,不能只切换二进制。发布说明列出可回滚边界与恢复步骤;禁止回滚时重新创建不确定是否存在的上游订阅。 + +## 验证和完成条件 + +- Open Flow 公共包测试从 `packages/open-flow` 执行 `bun run test`,不用 `bun test`。 +- Server 执行受影响的 publication、poll、integration、event source、恢复与隔离测试;按共享边界扩大验证范围。 +- 影响公开包代码、类型或资源时,从仓库根目录执行 `bun run --filter @oomol-lab/open-flow test:package`。 +- 每次提交前从根目录执行 `bun run check`;另外两个仓库使用其届时有效的检查与包验证命令。 +- 修改节点 authoring 时执行受影响 CLI/MCP Lab reference 场景;修改界面时先使用 frontend-ui skill,并在 Lab 检查真实组件及组合状态。 +- 两个 Connector 分别进行已接入 Provider 的真实账号验收。Slack 覆盖创建与握手、重复投递、重启、撤权后的使用拒绝、修改秘密后重新验证和删除;续租、远端注销仅在 Provider 实际存在这些资源时验证。Mock 不能代替第三方协议证据。 + +整体完成时,新普通 Provider 不再要求 Open Flow 增加专用协议分支;Source 状态只有一个持久化权威;两个后端通过同一行为契约;分页不改变新事件型 Flow 的业务输入;现有已发布 Flow 的迁移和恢复路径可验证。 + +## 本轮交付与验收边界 + +- 两个 Connector 共用公开 connect 的行为 fixtures,覆盖全部 Source 创建/读取/编辑/删除,Slack 验签、共享状态及权限,Jira 创建响应丢失恢复和资源清理,Asana 创建期间握手,Tally 与 GitHub 现有协议;36 个目录逐项对照。 +- 共享应用实测 1/10/100 Sources:只匹配一个 Source 的普通回调均为 2 次 Connector 调用;挑战只需路由。Connector 内复用 Provider 的既有验签和来源匹配函数,CPU 仍随 Source 数量增长,没有额外索引或路由框架。来源凭据只在匹配后的处理阶段检查。 + +暂不重写 scheduler,不引入消息中间件、通用批处理窗口或 Provider 公共框架。 + +## 相关实现与规范 + +- [现有架构](../architecture.md#publication-and-trigger) +- [Trigger 权限与执行](../control/trigger-permissions.md) +- [共享事件源](shared-event-sources.md) +- [Node authoring](../authoring/node-authoring.md) +- [远端 Trigger 适配](../../packages/open-flow/src/trigger/providers/remote.ts) +- [Poll 输出契约](../../packages/open-flow/src/trigger/common/poll.ts) +- [Source 运行时](../../apps/server/node/runtime/event-source-runtime.ts) +- [Source 持久化与投递](../../apps/server/node/storage/event-source-store.ts) + +相邻仓库的主要入口为 connect 的 `src/triggers/`、`src/providers/`、`scripts/export-flow-trigger-catalog.ts`,以及 oomol-connector 的 `src/flow-triggers/`、`src/proxy/` 和 `scripts/flow-triggers/`。 + +## 自动化验收记录(2026-10-27) + +- Open Flow 公共包:237 个测试文件、2,952 个测试通过。 +- Server:全量 95 个测试文件、1,203 个测试通过;包含 Feishu 资源响应丢失恢复测试,确认 Open Flow 不再用旧的 uncertain 分支阻止 Connector 重试。 +- connect:Trigger 55 个测试、SQLite/PostgreSQL 与持久化 Source 99 个测试通过;fix-check 与 runtime package 构建通过。 +- 托管 Connector:flow-triggers 203 个测试、持久化 Source 测试通过;fix-check:full 通过。 +- 双后端共同协议、36 个目录及 Open Flow 纵向验收通过;1/10/100 Source 路由测试包括不可用的无关来源。 +- 本轮没有改视觉布局,Workbench 端口与说明通过公共目录、presentation/localization 测试验证;没有将静态测试或 mock 计为真实第三方账号验收。 diff --git a/docs/triggers/connector-contract.md b/docs/triggers/connector-contract.md new file mode 100644 index 00000000..ca92f21b --- /dev/null +++ b/docs/triggers/connector-contract.md @@ -0,0 +1,143 @@ +# Connector Trigger 操作契约与首批验收 + +本契约覆盖两个 Connector 与 Open Flow 的全部 7 个 Source Provider 引用操作、声明式回调握手、Poll 批量契约。真实账号验收由用户执行,当前未部署。 + +## 操作边界 + +客户端通过 `POST /v1/providers/:service/triggers/:triggerId/execute` 调用注册操作,由 Connector 构造第三方请求并执行凭据访问。未知操作和未知请求字段拒绝执行,不回落到通用 proxy。 + +| 操作 | 调用方提供 | Connector 返回和负责的行为 | +| ----------- | --------------------------------------------------------------------- | ---------------------------------------------------------------- | +| `options` | 业务配置、字段名 | 配置选项;不能创建订阅或 Run | +| `read` | 业务配置、上次成功提交的 checkpoint | 下一页或下一次观察;不得替 Open Flow 提交业务消费进度 | +| `reconcile` | 业务配置、callback URL、稳定 requestKey、active、可选 subscriptionId | 准备、维护或清理远端订阅;只暴露不透明订阅身份 | +| `receive` | subscriptionId、原始 callback 字节及受控 headers/query、admit/current | 校验订阅关联和第三方签名,返回事件、唤醒、忽略或 HTTP 响应 | +| `resource` | 业务配置、稳定 requestKey、active | 维护共享资源需求与清理 | +| `source` | initialize/update/inspect/prepare/receive 及现有 Source 快照 | 当前仍是 Source 协议计算;不能据此宣称 Connector 已持久化 Source | + +同一有效订阅的 requestKey 绑定不可变业务配置和 callback URL。创建响应丢失后通过持久化状态与上游资源关联恢复,不能直接重建;清理失败保留资源身份,重复成功清理不重复执行上游删除。 + +`receive` 的外层 200 表示 Connector 已处理操作,内部 `respond.status` 才是应返回第三方的 HTTP 状态。不能把无效签名产生的内部 401 当作可准入事件,也不能混淆为调用 Connector 的认证失败。 + +`connector_subscription` 关联当前订阅创建世代;缺失或不匹配时返回内部 404。它不能代替 Provider 签名或调用方订阅所有权校验。订阅删除后回调被忽略,不能恢复活动状态。 + +## 身份与进度 + +Tally 表单事件使用上游 `eventId` 作为业务去重键。相同回调重试必须产生相同的输出和身份;Connector 返回重复事件是允许的,Open Flow 负责去重准入。 + +GitHub Watch PR 观察当前状态,不保证观察到每一次中间变化。checkpoint 包含状态摘要和递增序号;没有变化时输出为 null。有变化时返回新 checkpoint 和 `sequence:version` 身份,调用方重复使用旧 checkpoint 必须得到同一次观察结果。状态 A → B → A 的第二次 A 是新的观察,不能被首次 A 的摘要去重掉。 + +Open Flow 在准入事务成功后推进业务 checkpoint。容量拒绝或提交失败时继续持有旧 checkpoint,后续读取允许重放同一观察。Connector 订阅初始化保存的 baseline 不等同于 Open Flow 后续消费进度。 + +Poll 保持 definitionVersion 2 和 `events[]` 输出。每个非空去重结果页创建一个批量 Run,页内多条事件一起交给下游,保留原有聚合、执行次数和成本语义。Run 准入成功后才提交 checkpoint,容量不足时保留旧 checkpoint 以便重试。首轮 baseline、重叠读取去重、候选与当前版本 fencing 保持原保证;本轮不引入单事件模式或定义升级路径。 + +## Slack 共享应用回调 + +共同用例通过公开 `source` 操作初始化同一应用的两个 workspace Source,检查身份不同且事件只匹配对应 workspace、应用和机器人安装。签名验证使用原始字节,覆盖带空白和 Unicode 的 JSON、大小写 header、正文被重序列化、过期时间戳、缺失签名及重复大小写 header。 + +有效 challenge 返回原始 challenge 并建立 readiness。事件按 event type 和 channel 匹配消费者;Slack 重试 header 不改变规范化事件与 `event_id`。不匹配来源返回成功的空响应,无效签名返回内部 401,均不能产生事件。 + +这些用例固定现有无状态 Source 协议,不证明两个 Source 已在 Connector 共用一条应用记录,也不证明跨 Team 授权、共享秘密轮换或本地 Run 去重。它们是阶段三调整状态所有权前的回归基线。 + +## 持久化 Source + +通过原注册操作入口 `POST /v1/providers/slackbot/triggers/slackbot.on_event/execute` 调用新的 `operation: "sourceReference"`。这是真实管理入口,直接读写正式应用状态,没有候选快照导入或暂存阶段。同一操作适用于 Feishu、Slack、Intercom、Sentry、Zoom、Fireflies、Jira。共享入口按 owner、deploymentId、Trigger key、appId 隔离;无共享应用入口的 Source 按 requestKey 独立持久化。 + +创建请求示例(连接仍由现有 HTTP 连接选择器指定): + +```json +{ + "operation": "sourceReference", + "request": { + "action": "create", + "deploymentId": "open-flow-production", + "requestKey": "source-create-unique-id", + "input": { + "appId": "A123", + "workspaceId": "T123", + "signingSecret": "replace-with-signing-secret", + "eventTypes": ["app_mention"] + } + } +} +``` + +创建返回 `source: { reference: { applicationId, sourceId }, revision, eventTypes, view, ready, expiresAt? }`。引用的字段值是不透明身份,不能作为授权凭据,也不能由客户端自行构造。后续请求带相同的 `deploymentId` 和 `reference`;创建结果未知时,删除的 `reference` 也可使用 `{ appId?, requestKey }`(共享应用入口要求 appId),由 Connector 在属主作用域内定位: + +| action | 其他输入 | 结果 | +| --------- | ------------------------------------------------------------------ | -------------------------------------------------------------------------------- | +| `get` | 无 | 当前公开 Source 视图与 readiness,不返回秘密或内部状态 | +| `prepare` | 消费者 `config`,包括 `eventTypes` | 检查共同验证状态、连接账号和事件类型;Slack 无托管远端订阅,返回空 subscriptions | +| `receive` | 原始 `rawBody`(base64)、`headers`、`consumers` | 从持久化配置验签,返回 challenge 响应或匹配消费者的规范化事件 | +| `update` | `expectedRevision`、`input.eventTypes`,可选 `input.signingSecret` | 条件更新事件类型及应用共享秘密,返回新的公开 Source 视图 | +| `delete` | 无 | 幂等返回 `{ "deleted": true }`;删除后读取和接收返回 404 | + +Source 的应用、工作区和连接身份不可修改;事件类型和共享秘密通过 `update` 修改。相同应用、相同 requestKey 重试返回同一引用;已有引用对应的配置或秘密冲突返回 409。不同应用下的 requestKey 分别命名。删除后保留不含秘密的 Source 记录,旧 create 重试返回 409,不复活已删除对象;重新创建必须使用新的 requestKey。调用方须在首次远程创建前持久化 requestKey,响应丢失后继续使用它。按创建身份删除时,即使创建尚未到达也写入删除标记,阻止迟到的创建请求复活资源;授权仍检查所选 Connection 及已存记录的归属,不需要上游凭据。 + +connect 将应用归属绑定到已认证的 persistent runtime token;环境级临时 token 无法使用此操作。托管 Connector 只接受可信 Team workload,并绑定 Team 主体。创建、读取、修改、准备和回调检查当前解析出的连接及账号;删除按持久化归属和显式连接 ID 校验,不读取或刷新上游凭据。connect 仍检查 token 的 Trigger 与连接权限,托管侧仍要求可信 Team token 并检查随请求提供的 accessGrant;Source ID、请求中的业务字段和 deploymentId 都不能替代这些授权检查。`deploymentId` 是可信调用方选择的命名空间,不是独立认证身份;复用同一凭据的调用方具有该凭据的权限,不能把不同字符串当作额外的权限隔离。 + +同一归属、deployment、应用下的 Source 共用一份秘密和验证状态。一次有效 challenge 更新整个应用的 readiness;新建同应用 Source 必须使用相同秘密。删除一个 Source 保留其他 Source 的状态,删除最后一个清空应用的当前秘密与验证状态。之后用新 requestKey、新秘密创建的 Source 必须重新验证。活动应用可以通过 `update` 修改共享秘密,所有关联 Source 立即失去验证状态,必须使用新秘密完成 challenge;不提供无中断轮换或双密钥过渡。 + +`revision` 是应用记录现有的数据库版本,由同应用的 Sources 共用;创建、删除、配置修改和 challenge 写入都可能推进它。`update` 使用最近读取的 `expectedRevision`,实际需要修改且版本已变化时返回 409,调用方重新读取后决定是否重试。请求中的事件类型和可选秘密已经等于当前值时直接返回当前视图,允许响应丢失后的重复提交;不会再次写入或清空后来完成的验证。它不保证重放历史结果,也不新增操作日志或查询服务。修改事件类型只影响指定 Source,修改秘密影响整个应用;名称和启用状态不属于此接口。 + +应用以一条经过现有 SecretCodec/SecretCipher 编码的记录持久化,内部含共享状态和 Source 集合,最多 100 个活动 Source。更新使用数据库 revision 比较并交换;并发创建、删除或 challenge 发生冲突时重新读取,持续冲突返回 409。旧 challenge 的在途结果不能恢复已经删除并重建的应用状态。正常事件不重写未变化的状态,但返回前复核读取版本。接收端只输出 Source 配置中声明的事件类型。 + +connect 的 SQLite、PostgreSQL、D1 共享存储实现;SQLite 和 PostgreSQL 的存储密钥重加密及完整 runtime reset 包含新表。托管 Connector 使用真实 Postgres store。connect 延续部署选择的 codec,明文 codec 不会自动变成加密存储。最后一个 Source 删除不代表历史备份中的秘密已经擦除。 + +Open Flow 已接入全部 7 个 Provider 的引用操作。Trigger 传输使用双方均支持的 `appId` 查询参数指定连接,删除由有管理权限的 operator 发起,客户端不再先要求该连接仍存在或有效。Connector 保留原属主隔离,其他属主无法删除。未知操作明确失败,不回退到传递整份 Source 或秘密。正式 Open Flow 路径不再使用旧 `source` 操作;Connector 保留该已发布 wire 契约。 + +### Open Flow 的持久化与重试 + +Server 迁移 54 增加稳定的部署命名空间,以及现有 Source 行上的 Connector 引用和未完成操作字段。Source 本地 ID 同时作为首次创建的 requestKey;创建前即持久化,响应丢失或重启后沿用。Source 保存公开视图缓存和应用/工作区路由信息,`secrets_json` 始终为空,不写入 Slack 的 `event_source_apps` 记录;请求指纹是摘要,不保存原始秘密。 + +名称和启用状态仅在本地修改。Provider 配置修改与删除先在本地事务内检查 revision、当前和准备中消费者,再记录未完成操作;同应用的冲突操作、发布和新的回调准入暂时拒绝。远端确认后提交本地结果;明确拒绝的编辑解除未完成状态,结果未知则保留状态,调用方以相同内容重试。相同请求可在重启后继续,无后台任务自动补交秘密。未完成创建需重试原请求后才能编辑,也可直接删除;未完成编辑允许删除接管,但仍受消费者约束。删除通过 Connector 确认后才移除本地记录,失败可重试,无远端资源时连接失效不阻断引用删除;尚有托管资源时必须完成远端清理,失败保留引用供恢复。创建失败保留本地记录供重试或删除,避免并发失败误删另一请求已完成的 Source;迟到的创建/编辑响应不得覆盖删除意图。 + +普通列表读取、发布准备及回调从 Connector 获取公开投影。修改秘密后清除其他关联 Source 的投影缓存,并推进本地 revision 以拒绝旧在途结果;新的验证状态只从 Connector 读取。事件仍通过现有持久化投递、去重和 Run 准入链路。 + +没有无中断密钥轮换、双密钥过渡、独立撤权清理系统或删除记录回收。上线前复查 Source 存量,迁移遇到任何旧 Source 时整笔回滚并明确拒绝,不能据目标 production 为空而丢弃其他部署的数据。 + +## 错误与版本 + +| 错误 | 操作 HTTP 状态 | 调用方行为 | +| -------------------------- | -------------- | ---------------------------------------------- | +| `invalid_input` | 400 | 修正操作、配置或 checkpoint;不是无限重试 | +| `invalid_input` 配置冲突 | 409 | 同一 requestKey 已绑定其他配置,不能覆盖 | +| `trigger_connection_error` | 409 | 检查连接与授权 | +| `proxy_upstream_error` | 503 | 保留进度并重试 | +| `provider_error` | 500 | 结果可能不确定,查询或幂等重试,不直接重建资源 | + +共同用例不要求两个部署具有相同的授权规则。connect 使用 persistent token、connection、verified account;托管 Connector 使用 Team、principal、app。授权隔离由各自已有测试负责。 + +当前操作请求没有 definition revision 或 protocol version 协商。未知版本字段被严格 schema 拒绝,只能证明输入拒绝行为。目录一致性检查也不能替代部署间版本协商;后续破坏性改动必须先实现可识别的兼容或拒绝边界。 + +## 可重复执行的检查 + +从 Open Flow 根目录执行,路径分别指向安装了依赖的两个源码 checkout: + +```sh +node scripts/check-trigger-contracts.ts /path/to/connect /path/to/oomol-connector +``` + +共同用例唯一源位于 connect 的 `contracts/triggers/conformance.ts`。Open Flow 脚本使用同一份用例运行两个适配器,再导出目录,比较全部 36 个 Trigger 在两个 Connector 与 Open Flow 内置目录中的完整声明。临时目录在成功或失败后清理,不更新生成文件。 + +connect 适配器使用真实 HTTP 路由、token 选择、TriggerRunner 和 SQLite 内存存储,替换上游 Provider transport。托管适配器使用真实 controller、请求解析、Provider 实现和订阅管理,替换 credential executor 并使用现有内存 store。托管适配器不证明完整授权链路;两个适配器均不证明数据库重启恢复或真实第三方行为。独立的存储测试另外验证 SQLite 文件数据库重开、Postgres 连接重建、存储密钥重加密和旧回调写入隔离;这些仍不替代真实 Slack 账号验收。 + +共同验收覆盖全部 Source 引用生命周期、Slack 事件类型编辑与共享秘密重新验证、Jira 托管资源创建响应丢失后的恢复及清理、Asana 创建期握手,以及 Tally/GitHub 既有操作。Slack 发布、投递和 Run 去重通过 Open Flow 纵向脚本验证;所有第三方 Provider 的真实账号行为仍需人工验收,不能将 mock 当成该证据。失败报告区分行为用例失败、目录不一致与环境/依赖失败。 + +### Slack 纵向验收 + +`bun scripts/check-trigger-contracts.ts ` 现在也会用两个适配器分别运行 `apps/server/scripts/check-trigger-source.ts`。脚本经过 Open Flow 的控制面、Source runtime、发布与 Run 准入,以及 Connector 的实际请求解析和 Source 实现,验证签名 challenge、重复事件、修改秘密、重新验证和连接失效后的删除。远端账号由适配器提供,托管适配器使用内存存储;这不替代真实 Slack 账号和真实数据库验证。 + +Server 的 Source 引用测试另行覆盖未完成操作的重启重试、秘密不落地、编辑与消费者约束、旧回调拒绝、连接失效后的未完成创建/编辑删除、迟到响应不覆盖删除、并发创建失败不误删成功记录和迁移拒绝;Connector 的 SQLite/PostgreSQL 测试覆盖正式状态持久化。 + +## 本轮新增操作与回调契约 + +`prepare` 接收消费者 config 及可选 endpointUrl,返回不透明资源 key。远端订阅配置只保存在 Connector;`resource {reference,key,active}` 复用现有资源订阅协调与恢复逻辑,返回 outcome 和公开 Source 投影。活动或状态不确定的资源必须先清理成功,才允许删除 Source。Open Flow 的 uncertain 状态记录待确认进度,不再阻止重试;资源幂等和恢复由 Connector 的已有订阅状态机负责。Jira 创建响应丢失后按已有远端回调地址查询,确认恢复后不重复创建。 + +`route {deploymentId,applicationId,headers,rawBody}` 只用于共享应用入口。Connector 验签并匹配来源,返回 reference 列表或 HTTP challenge/拒绝响应。路由不读取来源凭据;匹配后的 receive 仍验证连接、账号和消费者。路由授权仍限制应用归属及连接 grant。Open Flow 只向匹配且本地启用的 Source 投递,全部持久化成功后确认普通事件;匹配处理失败返回可重试错误。应用级 CAS 防止旧验签结果覆盖新秘密状态。 + +Integration 目录声明 `callback.headers/query`,Open Flow 只按声明转发,保留原始 body。Connector 发起远端创建前已有订阅身份,URL 带 `connector_reference` 与 nonce `connector_subscription`。Asana 的候选握手先验证订阅归属、active 与 nonce,再调用只返回 HTTP 响应的握手处理,不取得正在创建的订阅锁,不激活生产绑定。 + +### 路由测量 + +本机内存测试中,双后端分别在 1、10、100 个同应用 Source 下完成两次重复回调与投递检查。每次普通回调始终 2 次 Connector 调用;对应两次回调及检查总时长约为 connect 11.32/6.77/18.03 ms、托管适配器 5.55/5.30/14.02 ms。此数据衡量本地协议开销,不是生产网络延迟承诺。旧路径每次请求逐个调用 N 个来源;新路径为一次应用路由加匹配来源数,Connector 内签名/匹配遍历仍为 O(N)。额外 Source 的 receive 被模拟为连接不可用,目标事件仍成功,证明没有调用无关来源处理。 diff --git a/docs/triggers/release-baseline.md b/docs/triggers/release-baseline.md new file mode 100644 index 00000000..ad009e32 --- /dev/null +++ b/docs/triggers/release-baseline.md @@ -0,0 +1,75 @@ +# Trigger 改造发布基线 + +核实日期:2026 年 10 月 26 日。发布记录与本地 checkout 分开记录;以下发布版本不代表当前线上实例版本。 + +| 项目 | 已核实发布记录 | 本次实施 checkout | +| --------------- | --------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Open Flow | npm `next` 为 `0.1.0-beta.70`,registry gitHead 为 `45f7f5660295fc6280ca0ecaf474356eee60f929` | `376a92717d76d6bf484f1a87109143c24cb60e23` | +| connect | npm `1.9.3`;GitHub `v1.9.3` 发布目标为 `c7149e45f89d0534353b581a9e28707a54eda538` | `ac7d4d877e968d9e4d02177aa6cffa6fbf52c45e` | +| oomol-connector | GitHub `v0.0.447` 发布目标为 `3469c01d017b17aad8ea558b097780b6a4707aad` | `69bfca136f728dd19c22d65c3a4ef2113f83f7ef`,提交标题为 release v0.0.445 | + +通过 npm registry 下载并校验了两个公开 tarball 的 SHA-512 integrity: + +- [Open Flow beta.70](https://registry.npmjs.org/@oomol-lab/open-flow/-/open-flow-0.1.0-beta.70.tgz):`sha512-f7KnVRZs90TTQzw9czdXVvjq+4DYX7rh0Sb4dX7Hmo7IcKSSfJA2TrEJDfBgPf4dGfNRLrVsl0t5uQL8mY0pHg==`。实际产物的 runtime-contract 与声明文件包含 Engine v7;不能再把 v7 视为尚未发布的可自由复用边界。 +- [OpenConnector 1.9.3](https://registry.npmjs.org/@oomol-lab/open-connector/-/open-connector-1.9.3.tgz):`sha512-jA203jyblaGBfJV6yNb1aSUdZLokHUCLjGrBd7WFLSYWToMBwWQfP+5zc1Emk+hjzZdrdBg8Mn1KmJewiEaNrA==`。实际产物含注册 Trigger 操作与 Source operation;npm 元数据未提供 gitHead,GitHub 发布目标不能被当成已经验证的 tarball 源码映射。 + +GitHub 记录见 [OpenConnector v1.9.3](https://github.com/oomol-lab/open-connector/releases/tag/v1.9.3) 与 [托管 Connector v0.0.447](https://github.com/oomol/oomol-connector/releases/tag/v0.0.447)。托管镜像 digest、已部署实例版本及其数据库迁移状态尚未核实;本地 checkout 落后于最新发布目标,不修改或切换用户分支来掩盖该差异。 + +## 首批变更的兼容结论 + +本批次增加共同协议验收、后端适配与目录比较,并修复托管 Connector 对已生成 callback nonce 缺少校验的问题。没有改变 Open Flow 节点输出、Flow model、Engine 或 checkpoint 格式,因此不分配新的 Open Flow 契约版本。 + +nonce 校验收紧了托管 Connector 的回调接收:通过正常创建得到的 URL 已含 `connector_subscription`,Open Flow 现有适配会转发它;遗漏或不匹配的回调被拒绝。部署前需确认现有入口和中间代理保留此 query,不能通过重建已有远端订阅来修复代理丢参数。 + +## 后续破坏性改动的前置证据 + +正式 Source 管理和单事件 Run 上线前,重新核实届时最新发布产物,并取得目标部署的镜像/包版本、数据库迁移状态和活动 Publication/Run 的契约分布。对比最新发布目标与工作分支中受影响的 Trigger 实现;当前源码验收通过不证明未检查的已发布托管镜像行为相同。 + +根据 [contract evolution](../control/contracts/compatibility.md#release-baselines-and-contract-versioning) 分别决定协议、模型、Engine 与存储的兼容或拒绝边界。不能沿用旧文档中的 beta.69 基线,也不能因为本批次不改版本,就认为后续可在已发布 v7 中改变 Run 语义。 + +## Production Source 存量核验 + +核验时间:2026-10-27 10:31(Asia/Shanghai)。使用 Cloud 部署仓库的 production Wrangler 配置和 `oomol` profile,直接对远端 D1 执行只读计数查询。Directory 返回全部 tenant shards 为 `01`、`02`,分别对应 `open-flow-cloud-tenant-01`、`open-flow-cloud-tenant-02`,两个分片均已检查。 + +| 数据 | 分片 01 | 分片 02 | +| ---------------------------------------------------------- | ------: | ------: | +| `event_sources`,包含软删除记录 | 0 | 0 | +| `source_events` | 0 | 0 | +| `source_deliveries` | 0 | 0 | +| `source_subscriptions` | 0 | 0 | +| `source_preparations` | 0 | 0 | +| `source_event_versions` | 0 | 0 | +| `flow_trigger_bindings` 中配置引用 `sourceId.value` 的记录 | 0 | 0 | +| 其中 `current_publication_id` 非空的记录 | 0 | 0 | + +查询返回 `rows_written: 0`、`changed_db: false`。未读取秘密或事件正文,未执行生产迁移。此结论针对 Source 事件源,不代表线上没有其他 Trigger,也不证明其他自托管或开发环境没有 Source 数据。 + +据此移除尚未发布的 Source 导入暂存表、存储接口、专属测试和文档。正式 Source 管理直接面向新建数据实现;发布前复查上述存量即可。若届时出现数据,再按实际需求处理,不预建迁移系统。批量 Poll Flow 的显式语义升级属于另一项兼容义务,不因 Source 为空而取消。 + +## Slack 正式引用操作批次 + +两个 Connector 新增 `sourceReference` 操作,直接持久化应用共享状态及 Source 集合。connect 新增 `0019_trigger_sources.sql`(SQLite/D1、PostgreSQL),托管 Connector 新增 `0023_trigger_sources.sql`;均为新表,不转换或删除既有 Source 数据,也没有导入暂存表。托管 schema snapshot 保留历史 Trigger 表,迁移 SQL 不包含无关删除。 + +现有 `source` 请求和目录声明保持兼容,新操作在旧后端应明确拒绝。Open Flow 消费端、公共包、Engine 和 Run 契约本批未切换;正式客户端接入与上线验证仍待完成。自定义 Connector TriggerStore 实现需要补齐应用状态读取和 revision 条件写入接口;内置数据库适配器已接入并验证。此批尚未提交或部署,不能将本地测试通过当作生产已经支持引用操作。 + +## Slack Open Flow 引用接入批次 + +Open Flow Server 的 Slack Source 已使用 `sourceReference`,公共包新增引用操作与响应校验契约;现有 Control API 请求字段、Flow model、Engine、节点输出和 Run 语义不变,不分配新的模型或 Engine 版本。Server schema 升到 54,新增部署命名空间及 Source 引用/未完成操作字段。它明确拒绝有旧 Slack Source 的数据库,拒绝时保留数据和原 schema version;其他 Provider 的存储继续保留。 + +先部署支持创建、修改、引用回调和无凭据删除(包括按创建请求身份删除并拒绝迟到创建)的 Connector,再升级 Open Flow Server 与公共包。升级前停旧 Server、备份 SQLite(含 WAL)并复查 Slack Source 存量;若不为空,停止升级并根据真实数据决定处理方式。迁移后的数据库不能交给旧 Server。新建 Source 后,回滚也必须处理 Connector 上的正式引用,不能只回滚本地数据库或二进制。此批没有执行生产迁移或部署。 + +## Full Provider delivery and Poll scope correction (2026-10-27) + +本轮统一覆盖全部 7 个 Source。迁移 54 尚未发布,复用该迁移并将空存量前置条件扩大到全部 Source;不再保留旧 Source 的 Open Flow 执行分支。Connector 继续支持已发布的无状态 `source` wire 操作,正式 Open Flow 客户端只用 `sourceReference`。新增 Source 后不得把数据库交回旧 Server;回滚必须同时考虑远端引用和托管资源。 + +Poll delivery remains definitionVersion 2 with `events[]` and batch Runs. The proposed v3 +single-event contract, compatibility snapshots and explicit Trigger refresh upgrade are withdrawn. +Flow model 10, Engine v7 and checkpoint formats remain unchanged. + +At correction time, Open Flow PR #160 is unmerged. Connector changes have merged and need follow-up +fixes restoring batch definitions. The latest verified formal releases predate those changes; +main-branch images may already contain the changed catalog. Deploy a version containing the batch +restoration before upgrading Open Flow. Existing Poll Publications need no output remapping. + +Source deployment still requires the reference/resource/routing protocol and an empty legacy Source +store before migration 54. Real-account acceptance remains manual; no production deployment was run. diff --git a/packages/open-flow/src/connector/common/proxy.ts b/packages/open-flow/src/connector/common/proxy.ts index 8231dfae..ad0e8a45 100644 --- a/packages/open-flow/src/connector/common/proxy.ts +++ b/packages/open-flow/src/connector/common/proxy.ts @@ -1,6 +1,7 @@ -import type { SourceOperation } from '../../event-source/common/definition.ts' +import type { SourceOperation, SourceReferenceOperation } from '../../event-source/common/definition.ts' import type { JsonValue } from '../../flow/common/change.ts' export type TriggerOperationRequest = + | { readonly operation: 'sourceReference'; readonly request: SourceReferenceOperation } | { readonly operation: 'source'; readonly request: SourceOperation } | { readonly operation: 'read'; readonly config: Readonly>; readonly checkpoint: JsonValue } | { readonly operation: 'options'; readonly config: Readonly>; readonly field: string } diff --git a/packages/open-flow/src/event-source/common/definition.ts b/packages/open-flow/src/event-source/common/definition.ts index a7405712..03d5a082 100644 --- a/packages/open-flow/src/event-source/common/definition.ts +++ b/packages/open-flow/src/event-source/common/definition.ts @@ -40,3 +40,27 @@ export type SourceOperation = } export class SourceIdentityError extends Error {} + +export interface SourceReference { + readonly applicationId: string + readonly sourceId: string +} +export type SourceReferenceOperation = { readonly deploymentId: string } & ( + | { readonly action: 'route'; readonly applicationId: string; readonly headers: Readonly>; readonly rawBody: string } + | { + readonly action: 'create' + readonly requestKey: string + readonly input: Readonly> + } + | { readonly action: 'get'; readonly reference: SourceReference } + | { readonly action: 'delete'; readonly reference: SourceReference | { readonly appId?: string; readonly requestKey: string } } + | { + readonly action: 'update' + readonly reference: SourceReference + readonly expectedRevision: number + readonly input: Readonly> + } + | { readonly action: 'prepare'; readonly reference: SourceReference; readonly config: Readonly>; readonly endpointUrl?: string } + | { readonly action: 'resource'; readonly reference: SourceReference; readonly key: string; readonly active: boolean } + | ({ readonly action: 'receive'; readonly reference: SourceReference } & Omit, 'source' | 'action'>) +) diff --git a/packages/open-flow/src/event-source/common/index.ts b/packages/open-flow/src/event-source/common/index.ts index 8be15e45..1342c189 100644 --- a/packages/open-flow/src/event-source/common/index.ts +++ b/packages/open-flow/src/event-source/common/index.ts @@ -5,7 +5,15 @@ import { eventSourceViewSchema } from '../../control/common/eventSources.ts' import catalog from './catalog.generated.json' with { type: 'json' } import { SourceIdentityError } from './definition.ts' -export type { EventSourceDefinition, SourceConfiguration, SourceEvent, SourceSubscription, SourceOperation } from './definition.ts' +export type { + EventSourceDefinition, + SourceConfiguration, + SourceEvent, + SourceSubscription, + SourceOperation, + SourceReference, + SourceReferenceOperation, +} from './definition.ts' export { SourceIdentityError } from './definition.ts' export function eventSourceDefinition(kind: string): EventSourceDefinition { @@ -67,3 +75,25 @@ export const sourceReceiptSchema = z export type SourceResult = z.infer export type PreparedSource = z.infer export type SourceReceipt = z.infer + +export const sourceReferenceSchema = z.object({ applicationId: z.string().min(1), sourceId: z.string().min(1) }) +export const sourceReferenceSnapshotSchema = projectionSchema.extend({ + reference: sourceReferenceSchema, + revision: z.number().int().positive(), + eventTypes: z.array(z.string()), +}) +export const sourceReferenceResultSchema = z.object({ source: sourceReferenceSnapshotSchema }) +export const sourceReferenceDeleteResultSchema = z.object({ deleted: z.literal(true) }) +export const sourceReferenceResourceResultSchema = sourceReferenceResultSchema.extend({ outcome: z.enum(['ready', 'pending']) }) +export const sourceReferenceRouteResultSchema = z.object({ + references: z.array(sourceReferenceSchema), + response: sourceReceiptSchema.shape.response, +}) +export const preparedSourceReferenceSchema = sourceReferenceResultSchema.extend({ subscriptions: z.array(z.object({ key: z.string() })) }) +export const sourceReferenceReceiptSchema = z + .object({ + ...sourceReceiptSchema.shape, + source: sourceReferenceSnapshotSchema.optional(), + }) + .refine((value) => (value.response == null) !== (value.event == null)) +export type SourceReferenceSnapshot = z.infer diff --git a/packages/open-flow/src/trigger/providers/catalog.generated.json b/packages/open-flow/src/trigger/providers/catalog.generated.json index f68842c9..d87832fd 100644 --- a/packages/open-flow/src/trigger/providers/catalog.generated.json +++ b/packages/open-flow/src/trigger/providers/catalog.generated.json @@ -204,6 +204,10 @@ } ] }, + "callback": { + "headers": ["x-hook-secret", "x-hook-signature"], + "query": [] + }, "options": false }, { @@ -283,6 +287,10 @@ } ] }, + "callback": { + "headers": ["calendly-webhook-signature"], + "query": [] + }, "options": false }, { @@ -363,6 +371,10 @@ } ] }, + "callback": { + "headers": ["x-signature"], + "query": [] + }, "options": false }, { @@ -495,8 +507,6 @@ } ] }, - "options": false, - "eventSource": "feishu", "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -613,7 +623,9 @@ "maximumBodyBytes": 65536, "applicationIngress": false, "managed": false - } + }, + "options": false, + "eventSource": "feishu" }, { "snapshot": { @@ -698,8 +710,10 @@ } ] }, - "options": false, - "eventSource": "fireflies", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -808,7 +822,9 @@ "maximumBodyBytes": 65536, "applicationIngress": false, "managed": false - } + }, + "options": false, + "eventSource": "fireflies" }, { "snapshot": { @@ -881,6 +897,14 @@ }, "nullable": false }, + { + "handle": "insecureSsl", + "jsonSchema": { + "type": "boolean" + }, + "nullable": false, + "value": false + }, { "handle": "owner", "jsonSchema": { @@ -896,18 +920,6 @@ "pattern": "^(?!\\.{1,2}$)[a-zA-Z0-9._-]{1,100}$" }, "nullable": false - }, - { - "group": "Advanced settings", - "collapsed": true - }, - { - "handle": "insecureSsl", - "jsonSchema": { - "type": "boolean" - }, - "nullable": false, - "value": false } ], "definitionVersion": 2, @@ -950,6 +962,10 @@ "provider": "github", "type": "integration" }, + "callback": { + "headers": ["x-github-delivery", "x-github-event", "x-hub-signature-256"], + "query": [] + }, "options": false }, { @@ -1049,6 +1065,10 @@ } ] }, + "callback": { + "headers": ["x-github-delivery", "x-github-event", "x-hub-signature-256"], + "query": [] + }, "options": false, "intervalMs": 300000 }, @@ -1086,6 +1106,14 @@ }, "nullable": false }, + { + "handle": "insecureSsl", + "jsonSchema": { + "type": "boolean" + }, + "nullable": false, + "value": false + }, { "handle": "project", "jsonSchema": { @@ -1104,18 +1132,6 @@ }, "nullable": false, "value": "" - }, - { - "group": "Advanced settings", - "collapsed": true - }, - { - "handle": "insecureSsl", - "jsonSchema": { - "type": "boolean" - }, - "nullable": false, - "value": false } ], "definitionVersion": 2, @@ -1165,6 +1181,10 @@ "provider": "gitlab", "type": "integration" }, + "callback": { + "headers": ["idempotency-key", "x-gitlab-event", "x-gitlab-token"], + "query": [] + }, "options": false }, { @@ -1462,44 +1482,31 @@ "description": "Optional shared drive ID. Omit it to monitor the connected account." }, { - "handle": "includeRemoved", - "jsonSchema": { - "type": "boolean" - }, - "nullable": false, - "value": true, - "description": "Include changes caused by deletion or loss of access." - }, - { - "handle": "restrictToMyDrive", + "handle": "includeCorpusRemovals", "jsonSchema": { "type": "boolean" }, "nullable": false, "value": false, - "description": "Restrict changes to the My Drive hierarchy." - }, - { - "group": "Advanced settings", - "collapsed": true + "description": "Include the accessible file resource when an item leaves the change corpus." }, { - "handle": "includeCorpusRemovals", + "handle": "includeItemsFromAllDrives", "jsonSchema": { "type": "boolean" }, "nullable": false, - "value": false, - "description": "Include the accessible file resource when an item leaves the change corpus." + "value": true, + "description": "Include items from My Drive and shared drives." }, { - "handle": "includeItemsFromAllDrives", + "handle": "includeRemoved", "jsonSchema": { "type": "boolean" }, "nullable": false, "value": true, - "description": "Include items from My Drive and shared drives." + "description": "Include changes caused by deletion or loss of access." }, { "handle": "pageSize", @@ -1511,6 +1518,15 @@ "nullable": false, "value": 100, "description": "Maximum changes included in one Flow Run." + }, + { + "handle": "restrictToMyDrive", + "jsonSchema": { + "type": "boolean" + }, + "nullable": false, + "value": false, + "description": "Restrict changes to the My Drive hierarchy." } ], "definitionVersion": 2, @@ -1589,6 +1605,18 @@ "provider": "googledrive", "type": "integration" }, + "callback": { + "headers": [ + "x-goog-changed", + "x-goog-channel-id", + "x-goog-channel-token", + "x-goog-message-number", + "x-goog-resource-id", + "x-goog-resource-state", + "x-goog-resource-uri" + ], + "query": [] + }, "options": false }, { @@ -1761,44 +1789,31 @@ "description": "Optional shared drive ID. Omit it to monitor the connected account." }, { - "handle": "includeRemoved", - "jsonSchema": { - "type": "boolean" - }, - "nullable": false, - "value": true, - "description": "Include changes caused by deletion or loss of access." - }, - { - "handle": "restrictToMyDrive", + "handle": "includeCorpusRemovals", "jsonSchema": { "type": "boolean" }, "nullable": false, "value": false, - "description": "Restrict changes to the My Drive hierarchy." - }, - { - "group": "Advanced settings", - "collapsed": true + "description": "Include the accessible file resource when an item leaves the change corpus." }, { - "handle": "includeCorpusRemovals", + "handle": "includeItemsFromAllDrives", "jsonSchema": { "type": "boolean" }, "nullable": false, - "value": false, - "description": "Include the accessible file resource when an item leaves the change corpus." + "value": true, + "description": "Include items from My Drive and shared drives." }, { - "handle": "includeItemsFromAllDrives", + "handle": "includeRemoved", "jsonSchema": { "type": "boolean" }, "nullable": false, "value": true, - "description": "Include items from My Drive and shared drives." + "description": "Include changes caused by deletion or loss of access." }, { "handle": "pageSize", @@ -1810,6 +1825,15 @@ "nullable": false, "value": 100, "description": "Maximum changes included in one Flow Run." + }, + { + "handle": "restrictToMyDrive", + "jsonSchema": { + "type": "boolean" + }, + "nullable": false, + "value": false, + "description": "Restrict changes to the My Drive hierarchy." } ], "definitionVersion": 2, @@ -1841,6 +1865,18 @@ "provider": "googledrive", "type": "integration" }, + "callback": { + "headers": [ + "x-goog-changed", + "x-goog-channel-id", + "x-goog-channel-token", + "x-goog-message-number", + "x-goog-resource-id", + "x-goog-resource-state", + "x-goog-resource-uri" + ], + "query": [] + }, "options": false, "intervalMs": 300000 }, @@ -2053,8 +2089,10 @@ } ] }, - "options": false, - "eventSource": "intercom", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -2168,7 +2206,9 @@ "maximumBodyBytes": 65536, "applicationIngress": true, "managed": false - } + }, + "options": false, + "eventSource": "intercom" }, { "snapshot": { @@ -2253,8 +2293,10 @@ } ] }, - "options": false, - "eventSource": "jira", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -2363,7 +2405,9 @@ "maximumBodyBytes": 65536, "applicationIngress": false, "managed": true - } + }, + "options": false, + "eventSource": "jira" }, { "snapshot": { @@ -2933,6 +2977,10 @@ } ] }, + "callback": { + "headers": [], + "query": ["signature"] + }, "options": false }, { @@ -2997,6 +3045,10 @@ } ] }, + "callback": { + "headers": ["authorization"], + "query": [] + }, "options": false }, { @@ -3073,6 +3125,10 @@ } ] }, + "callback": { + "headers": ["svix-id", "svix-signature", "svix-timestamp"], + "query": [] + }, "options": false }, { @@ -3184,8 +3240,10 @@ } ] }, - "options": false, - "eventSource": "sentry", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -3309,7 +3367,9 @@ "maximumBodyBytes": 65536, "applicationIngress": true, "managed": false - } + }, + "options": false, + "eventSource": "sentry" }, { "snapshot": { @@ -3579,6 +3639,10 @@ "provider": "shopify", "type": "integration" }, + "callback": { + "headers": ["x-shopify-api-version", "x-shopify-event-id", "x-shopify-shop-domain", "x-shopify-topic", "x-shopify-triggered-at", "x-shopify-webhook-id"], + "query": ["open_flow_callback"] + }, "options": false }, { @@ -3860,8 +3924,10 @@ } ] }, - "options": false, - "eventSource": "slack", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -3978,11 +4044,22 @@ "maximumBodyBytes": 65536, "applicationIngress": true, "managed": false - } + }, + "options": false, + "eventSource": "slack" }, { "snapshot": { "configInputs": [ + { + "handle": "apiVersion", + "jsonSchema": { + "type": "string", + "maxLength": 40 + }, + "nullable": false, + "value": "" + }, { "handle": "events", "jsonSchema": { @@ -4004,19 +4081,6 @@ }, "nullable": false, "value": false - }, - { - "group": "Advanced settings", - "collapsed": true - }, - { - "handle": "apiVersion", - "jsonSchema": { - "type": "string", - "maxLength": 40 - }, - "nullable": false, - "value": "" } ], "definitionVersion": 2, @@ -4066,6 +4130,10 @@ "provider": "stripe", "type": "integration" }, + "callback": { + "headers": ["stripe-signature"], + "query": [] + }, "options": false }, { @@ -4152,6 +4220,10 @@ "provider": "tally", "type": "integration" }, + "callback": { + "headers": ["tally-signature"], + "query": [] + }, "options": true }, { @@ -4172,6 +4244,15 @@ "value": [], "description": "Only accept updates from these numeric Telegram chat IDs. Empty means every chat." }, + { + "handle": "dropPendingUpdates", + "jsonSchema": { + "type": "boolean" + }, + "nullable": false, + "value": true, + "description": "Discard updates queued before the webhook is created or resumed." + }, { "handle": "updates", "jsonSchema": { @@ -4227,19 +4308,6 @@ "nullable": false, "value": [], "description": "Only accept updates from these numeric Telegram user IDs. Empty means every user." - }, - { - "group": "Advanced settings", - "collapsed": true - }, - { - "handle": "dropPendingUpdates", - "jsonSchema": { - "type": "boolean" - }, - "nullable": false, - "value": true, - "description": "Discard updates queued before the webhook is created or resumed." } ], "definitionVersion": 2, @@ -4282,6 +4350,10 @@ "provider": "telegram", "type": "integration" }, + "callback": { + "headers": ["x-telegram-bot-api-secret-token"], + "query": [] + }, "options": false }, { @@ -4348,6 +4420,10 @@ } ] }, + "callback": { + "headers": ["typeform-signature"], + "query": [] + }, "options": false }, { @@ -4439,6 +4515,10 @@ } ] }, + "callback": { + "headers": ["x-vercel-signature"], + "query": [] + }, "options": false }, { @@ -4475,10 +4555,6 @@ }, "nullable": false }, - { - "group": "Advanced settings", - "collapsed": true - }, { "handle": "webhookName", "jsonSchema": { @@ -4558,6 +4634,18 @@ "provider": "woocommerce", "type": "integration" }, + "callback": { + "headers": [ + "x-wc-webhook-delivery-id", + "x-wc-webhook-event", + "x-wc-webhook-id", + "x-wc-webhook-resource", + "x-wc-webhook-signature", + "x-wc-webhook-source", + "x-wc-webhook-topic" + ], + "query": [] + }, "options": false }, { @@ -4695,6 +4783,10 @@ "provider": "zendesk", "type": "integration" }, + "callback": { + "headers": ["x-zendesk-webhook-signature", "x-zendesk-webhook-signature-timestamp"], + "query": [] + }, "options": false }, { @@ -4780,8 +4872,10 @@ } ] }, - "options": false, - "eventSource": "zoom", + "callback": { + "headers": [], + "query": [] + }, "source": { "createSchema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -4895,6 +4989,8 @@ "maximumBodyBytes": 65536, "applicationIngress": true, "managed": false - } + }, + "options": false, + "eventSource": "zoom" } ] diff --git a/packages/open-flow/src/trigger/providers/fieldDescriptions.ts b/packages/open-flow/src/trigger/providers/fieldDescriptions.ts index 8ac5b5fa..b586d50b 100644 --- a/packages/open-flow/src/trigger/providers/fieldDescriptions.ts +++ b/packages/open-flow/src/trigger/providers/fieldDescriptions.ts @@ -7,7 +7,7 @@ const driveChangesConfig = { includeCorpusRemovals: 'Include the accessible file resource when an item leaves the change corpus.', includeItemsFromAllDrives: 'Include items from My Drive and shared drives.', includeRemoved: 'Include changes caused by deletion or loss of access.', - pageSize: 'Maximum number of changes included in one Flow Run.', + pageSize: 'Maximum number of changes read per page.', restrictToMyDrive: 'Restrict changes to the My Drive hierarchy.', } diff --git a/packages/open-flow/src/trigger/providers/remote.ts b/packages/open-flow/src/trigger/providers/remote.ts index 03960a18..14d478e5 100644 --- a/packages/open-flow/src/trigger/providers/remote.ts +++ b/packages/open-flow/src/trigger/providers/remote.ts @@ -120,66 +120,20 @@ function createDefinition(item: (typeof catalog)[number]): PollDefinition | Inte return { outcome: result.outcome } as IntegrationReconcileResult }, async receive(context) { - if (snapshot.key === 'asana.on_task_event') { - const secret = context.header('x-hook-secret') - if (secret != null) { - if (context.method !== 'POST' || !/^[a-fA-F0-9]{32,256}$/.test(secret)) - return { outcome: 'respond', status: 400, body: '', contentType: 'text/plain' } - // Acknowledge before Connector creation returns; persist the secret only from its authenticated response. - return { outcome: 'respond', status: 200, body: '', contentType: 'text/plain', headers: { 'x-hook-secret': secret } } - } - } - const id = context.state?.subscription.id + const id = context.state?.subscription.id ?? context.query('connector_reference') if (typeof id !== 'string') return { outcome: 'respond', status: 503, body: '', contentType: 'text/plain' } const headers: Record = {} - for (const name of [ - 'content-type', - 'idempotency-key', - 'stripe-signature', - 'tally-signature', - 'typeform-signature', - 'x-signature', - 'svix-id', - 'svix-timestamp', - 'svix-signature', - 'x-vercel-signature', - 'calendly-webhook-signature', - 'authorization', - 'x-hook-signature', - 'x-github-delivery', - 'x-github-event', - 'x-gitlab-event', - 'x-gitlab-token', - 'x-goog-changed', - 'x-goog-channel-id', - 'x-goog-channel-token', - 'x-goog-message-number', - 'x-goog-resource-id', - 'x-goog-resource-state', - 'x-goog-resource-uri', - 'x-hub-signature-256', - 'x-shopify-api-version', - 'x-shopify-event-id', - 'x-shopify-shop-domain', - 'x-shopify-topic', - 'x-shopify-triggered-at', - 'x-shopify-webhook-id', - 'x-telegram-bot-api-secret-token', - 'x-wc-webhook-delivery-id', - 'x-wc-webhook-event', - 'x-wc-webhook-id', - 'x-wc-webhook-resource', - 'x-wc-webhook-signature', - 'x-wc-webhook-source', - 'x-wc-webhook-topic', - 'x-zendesk-webhook-signature', - 'x-zendesk-webhook-signature-timestamp', - ]) { + for (const name of ['content-type', ...('callback' in item ? (('callback' in item ? item.callback : undefined)?.headers ?? []) : [])]) { const value = context.header(name) if (value != null) headers[name] = value } const query: Record = {} - for (const name of ['open_flow_callback', 'connector_subscription', 'signature']) { + for (const name of [ + 'open_flow_callback', + 'connector_subscription', + 'connector_reference', + ...('callback' in item ? (('callback' in item ? item.callback : undefined)?.query ?? []) : []), + ]) { const value = context.query(name) if (value != null) query[name] = value } diff --git a/packages/open-flow/test/remote-trigger.test.ts b/packages/open-flow/test/remote-trigger.test.ts index 8046256a..9e5ef77b 100644 --- a/packages/open-flow/test/remote-trigger.test.ts +++ b/packages/open-flow/test/remote-trigger.test.ts @@ -114,7 +114,9 @@ it('rejects legacy remote identifiers before creating another subscription', asy it('acknowledges Asana creation handshakes before a subscription exists without accepting or persisting the supplied key', async () => { const definition = remoteTriggerDefinitions.find((item) => item.snapshot.key === 'asana.on_task_event') as IntegrationDefinition - const trigger = vi.fn() + const trigger = vi.fn(async () => ({ + result: { outcome: 'respond', status: 200, body: '', contentType: 'text/plain', headers: { 'x-hook-secret': 'a'.repeat(64) } }, + })) const saveSubscription = vi.fn() const secret = 'a'.repeat(64) const result = await definition.receive({ @@ -128,12 +130,15 @@ it('acknowledges Asana creation handshakes before a subscription exists without payload: null, rawBody: new Uint8Array(), header: (name) => (name === 'x-hook-secret' ? secret : undefined), - query: () => undefined, + query: (name) => (name === 'connector_reference' ? 'candidate-reference' : undefined), connector: { execute: vi.fn(), trigger }, state: { checkpoint: null, subscription: {}, saveCheckpoint: vi.fn(), saveSubscription }, }) expect(result).toEqual({ outcome: 'respond', status: 200, body: '', contentType: 'text/plain', headers: { 'x-hook-secret': secret } }) - expect(trigger).not.toHaveBeenCalled() + expect(trigger).toHaveBeenCalledWith( + expect.objectContaining({ operation: 'receive', subscriptionId: 'candidate-reference', headers: { 'x-hook-secret': secret } }), + undefined, + ) expect(saveSubscription).not.toHaveBeenCalled() }) diff --git a/scripts/check-trigger-contracts.ts b/scripts/check-trigger-contracts.ts new file mode 100644 index 00000000..035fb2b5 --- /dev/null +++ b/scripts/check-trigger-contracts.ts @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { resolve, join } from 'node:path' + +const [publicCheckout, managedCheckout] = process.argv.slice(2) +if (!publicCheckout || !managedCheckout) throw new Error('Pass the OpenConnector and managed Connector checkout paths.') +const publicRoot = resolve(publicCheckout) +const managedRoot = resolve(managedCheckout) +const root = resolve(import.meta.dirname, '..') +const suite = join(publicRoot, 'contracts/triggers/run.ts') +const directory = mkdtempSync(join(tmpdir(), 'trigger-contracts-')) +const run = (command: string, args: string[], cwd: string) => { + const result = spawnSync(command, args, { cwd, stdio: 'inherit' }) + if (result.error) throw result.error + if (result.status !== 0) throw new Error(`${command} ${args[0]} failed (${result.status ?? result.signal}).`) +} + +try { + run('node', [suite, 'src/server/trigger-conformance-adapter.ts'], publicRoot) + run('bun', [suite, 'scripts/flow-triggers/conformance-adapter.ts'], managedRoot) + const publicCatalog = join(directory, 'public.json') + const managedCatalog = join(directory, 'managed.json') + run('node', ['scripts/export-flow-trigger-catalog.ts', publicCatalog], publicRoot) + run('bun', ['scripts/flow-triggers/export-catalog.ts', managedCatalog], managedRoot) + const catalogs = [publicCatalog, managedCatalog, join(root, 'packages/open-flow/src/trigger/providers/catalog.generated.json')].map( + (file) => JSON.parse(readFileSync(file, 'utf8')) as { snapshot: { key: string } }[], + ) + for (const key of catalogs[0]!.map((entry) => entry.snapshot.key)) { + const entries = catalogs.map((catalog) => catalog.filter((entry) => entry.snapshot.key === key)) + entries.forEach((entry, index) => assert.equal(entry.length, 1, `${key}: catalog ${index} must contain one definition`)) + assert.deepEqual(entries[1], entries[0], `${key}: Connector catalog contracts differ`) + assert.deepEqual(entries[2], entries[0], `${key}: Open Flow's bundled catalog differs`) + console.log(`PASS ${key} catalog compatibility across both backends and Open Flow`) + } + for (const adapter of [join(publicRoot, 'src/server/trigger-conformance-adapter.ts'), join(managedRoot, 'scripts/flow-triggers/conformance-adapter.ts')]) { + run('node', ['--no-node-snapshot', '--import', '@oxc-node/core/register', 'apps/server/scripts/check-trigger-source.ts', adapter], root) + } +} finally { + rmSync(directory, { recursive: true, force: true }) +}