Skip to content
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/schedule-session-turn.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@truefoundry/trueforge": patch
---

Dispatch schedule runs through the session/turn API: get-or-create a session keyed by run id, then create a turn only when that session has none.
1 change: 1 addition & 0 deletions packages/trueforge/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
"@hono/zod-openapi": "^1.6.1",
"@modelcontextprotocol/sdk": "^1.30.0",
"@truefoundry/trueforge-core": "workspace:*",
"@truefoundry/trueforge-sdk": "workspace:*",
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
"better-sqlite3": "^13.0.3",
"cron-parser": "^5.4.0",
"env-paths": "^4.0.0",
Expand Down
109 changes: 41 additions & 68 deletions packages/trueforge/src/controller.ts
Original file line number Diff line number Diff line change
@@ -1,70 +1,47 @@
/**
* Controller entry point — the control loops as their own process.
*
* Distributed mode only. Server replicas run no loops, so this deployment is the sole
* controller for the database. Loops assume no peer runs alongside them,
* so it MUST have exactly one replica.
*
* In standalone mode the controller runs inside the server instead — one process owns
* everything, and this entry point refuses to start.
*/
import { extractErrorLogFields } from '@truefoundry/trueforge-core/core';

let configuration: typeof import('./config').default;

try {
({ default: configuration } = await import('./config'));
} catch (error) {
console.error(
'Failed to start controller: Failed to load configuration:',
error instanceof Error ? error.message : error,
);
process.exit(1);
}

import { TrueForge } from '@truefoundry/trueforge-sdk';
import type { Logger } from 'winston';
import { Controller } from './controller/Controller';
import { createScheduleDispatchLoop, logTriggeredRun } from './controller/scheduleDispatch';
import { createDb } from './db/postgres/client';
import { PostgresScheduleStore } from './db/postgres/schedule-store/PostgresScheduleStore';
import { createServerLogger } from './logger';
import { PACKAGE_VERSION } from './packageVersion';

try {
const logger = createServerLogger({
level: configuration.LOG_LEVEL,
standalone: configuration.STANDALONE,
version: PACKAGE_VERSION,
});
import { scheduleDispatchLoop } from './controller/scheduleDispatch';
import type { IScheduleStore } from './db/scheduleStore';
import type { WithTransaction } from './db/transaction';

// Narrow through a const: the config union is keyed on STANDALONE, and only the
// distributed variant carries the Postgres connection settings.
const config = configuration;
if (config.STANDALONE) {
throw new Error(
'The controller process is for distributed mode only. ' +
'In standalone mode the server runs the control loops in-process.',
);
}

const db = createDb({
connectionString: config.DATABASE_URL,
poolMax: config.DATABASE_POOL_MAX,
statementTimeoutMs: config.POSTGRES_STATEMENT_TIMEOUT_MS,
idleInTransactionSessionTimeoutMs: config.POSTGRES_IDLE_IN_TRANSACTION_SESSION_TIMEOUT_MS,
});
const controller = new Controller({
/**
* The loops the controller runs.
*/
export function createController<TTransaction>(params: {
scheduleStore: IScheduleStore<TTransaction>;
withTransaction: WithTransaction<TTransaction>;
logger: Logger;
baseUrl: string;
}): Controller {
const { scheduleStore, withTransaction, logger, baseUrl } = params;
return new Controller({
loops: [
createScheduleDispatchLoop({
scheduleStore: new PostgresScheduleStore(db),
withTransaction: callback => db.transaction().execute(callback),
onTriggered: logTriggeredRun(logger),
scheduleDispatchLoop({
scheduleStore,
client: new TrueForge({ baseUrl, auth: false }),
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
withTransaction,
logger,
}),
],
logger,
});
}

logger.info('TrueForge controller starting', { mode: 'distributed' });
/**
* Runs the controller: starts the loops and drains them on SIGTERM/SIGINT.
*/
export function runController<TTransaction>(params: {
scheduleStore: IScheduleStore<TTransaction>;
withTransaction: WithTransaction<TTransaction>;
logger: Logger;
baseUrl: string;
gracefulTimeoutSeconds: number;
/** Releases what the caller opened for the loops, e.g. its database pool. */
onStopped?: () => Promise<void>;
}): Controller {
const { logger, gracefulTimeoutSeconds, onStopped } = params;
const controller = createController(params);
controller.start();

let shuttingDown = false;
Expand All @@ -75,17 +52,14 @@ try {
shuttingDown = true;
logger.info(`Received ${signal}, stopping control loops`);

// Start the deadline at the top of shutdown; unref so this timer alone cannot keep the process
// alive. Passes only hold short transactions, so this should never elapse.
// Passes only hold short transactions, so the deadline should never elapse.
setTimeout(() => {
logger.warn(`Controller drain timed out after ${String(configuration.GRACEFUL_TIMEOUT_SECONDS)}s, exiting`);
logger.warn(`Controller drain timed out after ${String(gracefulTimeoutSeconds)}s, exiting`);
process.exit(1);
}, configuration.GRACEFUL_TIMEOUT_SECONDS * 1000).unref();
}, gracefulTimeoutSeconds * 1000).unref();

await controller.stop();
await db.destroy().catch((error: unknown) => {
logger.warn('Error closing the database pool during shutdown', extractErrorLogFields(error));
});
await onStopped?.();
process.exit(0);
};
process.on('SIGTERM', signal => {
Expand All @@ -94,7 +68,6 @@ try {
process.on('SIGINT', signal => {
void shutdown(signal);
});
} catch (error) {
console.error('Failed to start controller:', error instanceof Error ? error.message : error);
process.exit(1);

return controller;
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
}
2 changes: 1 addition & 1 deletion packages/trueforge/src/controller/Controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
*
* The controller runs in exactly ONE process per database. In standalone mode that is
* the server itself; when `STANDALONE=false` it is the dedicated single-replica
* process in `src/controller.ts`. Loops are written assuming this.
* process. Loops are written assuming this.
*/
import type { Logger } from 'winston';

Expand Down
58 changes: 33 additions & 25 deletions packages/trueforge/src/controller/scheduleDispatch.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { TrueForge } from '@truefoundry/trueforge-sdk';
import type { Logger } from 'winston';
import {
cronRunName,
Expand Down Expand Up @@ -31,6 +32,34 @@ const SCHEDULE_DISPATCH_INTERVAL_MS = 60_000;
/** The loop's name. */
const SCHEDULE_DISPATCH_LOOP_NAME = 'schedule-dispatch';

type ScheduleRunApiClient = Pick<TrueForge, 'sessions' | 'internal'>;

/**
* Hands a due schedule run to the API:
* 1. Get or create a session keyed by `run.id`.
* 2. Create a non-streaming turn only when that session has no turns.
*
* This call is hence idempotent.
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
*/
function executeScheduledRun(client: ScheduleRunApiClient): (item: ScheduleDispatchItem) => Promise<void> {
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
return async ({ run, schedule }) => {
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
const { data: session } = await client.internal.sessions.getOrCreateByExternalId({
externalId: run.id,
agent: { name: schedule.agent_name },
});
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

const turns = await client.sessions.listTurns(session.id, { limit: 1 });
if (turns.data.length > 0) {
return;
}

await client.sessions.createTurn(session.id, {
input: [{ type: 'user.message', content: schedule.manifest.task }],
previousTurnId: 'none',
});
};
Comment thread
heerambavi1998 marked this conversation as resolved.
Resolved
}

/**
* Mark the current run with `status`, then add the schedule's next scheduled run
* if the schedule is still active and its cron still has a later trigger time.
Expand Down Expand Up @@ -202,41 +231,20 @@ export async function dispatchScheduledRuns<TTransaction>(params: {
return { dispatched, failed };
}

/**
* Stand-in until the run executor lands: records the hand-off and drops it.
*
* Called before the row is marked `triggered`; if nothing commits afterward the
* same scheduled row will be offered again on the next pass.
*
* TODO(executor): replace with the real executor. Note it runs in the controller's
* process, so the dedicated deployment will need everything a turn needs — model
* provider credentials, MCP egress, sandbox access — not just database reach.
*/
export function logTriggeredRun(logger: Logger): (item: ScheduleDispatchItem) => void | Promise<void> {
return item => {
logger.warn('Schedule run triggered but no executor is configured; run will not execute', {
schedule_id: item.schedule.id,
run_id: item.run.id,
agent_name: item.schedule.agent_name,
scheduled_for: item.run.scheduled_for,
});
};
}

export function createScheduleDispatchLoop<TTransaction>(params: {
export function scheduleDispatchLoop<TTransaction>(params: {
scheduleStore: IScheduleStore<TTransaction>;
onTriggered: (item: ScheduleDispatchItem) => void | Promise<void>;
client: ScheduleRunApiClient;
logger: Logger;
withTransaction: WithTransaction<TTransaction>;
}): ControlLoop {
const { scheduleStore, withTransaction, onTriggered, logger } = params;
const { scheduleStore, client, withTransaction, logger } = params;
return {
name: SCHEDULE_DISPATCH_LOOP_NAME,
intervalMs: SCHEDULE_DISPATCH_INTERVAL_MS,
async tick(signal: AbortSignal): Promise<void> {
const result = await dispatchScheduledRuns({
store: scheduleStore,
onTriggered,
onTriggered: executeScheduledRun(client),
logger,
withTransaction,
signal,
Expand Down
2 changes: 2 additions & 0 deletions packages/trueforge/src/db/indexes.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/** Partial unique index on `session (tenant_id, external_id) WHERE external_id IS NOT NULL`. */
export const SESSION_EXTERNAL_ID_UQ = 'session_external_id_uq';
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { sql, type Kysely } from 'kysely';
import { SESSION_EXTERNAL_ID_UQ } from '../../indexes';

/**
* Optional session `external_id`, unique within a tenant when set.
Expand All @@ -8,14 +9,14 @@ export async function up(db: Kysely<unknown>): Promise<void> {
await sql`SET LOCAL lock_timeout = '5s'`.execute(db);
await db.schema.alterTable('session').addColumn('external_id', 'text').execute();
await sql`
CREATE UNIQUE INDEX ${sql.raw('session_external_id_uq')}
CREATE UNIQUE INDEX ${sql.raw(SESSION_EXTERNAL_ID_UQ)}
ON session (tenant_id, external_id)
WHERE external_id IS NOT NULL
`.execute(db);
}

export async function down(db: Kysely<unknown>): Promise<void> {
await sql`SET LOCAL lock_timeout = '5s'`.execute(db);
await sql`DROP INDEX IF EXISTS ${sql.raw('session_external_id_uq')}`.execute(db);
await sql`DROP INDEX IF EXISTS ${sql.raw(SESSION_EXTERNAL_ID_UQ)}`.execute(db);
await db.schema.alterTable('session').dropColumn('external_id').execute();
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
SessionStoreInvariantError,
} from '@truefoundry/trueforge-core/agent-session/store/SessionStoreErrors';
import { sql, type Kysely } from 'kysely';
import { SESSION_EXTERNAL_ID_UQ } from '../../../indexes';
import { sessionAgentFromColumns, sessionAgentToColumns } from '../../../sessionAgentColumns';
import { isPgConstraint, isUniqueViolation } from '../../client';
import { json } from '../../sqlExpressions';
Expand Down Expand Up @@ -107,7 +108,7 @@ export async function createSession(db: Kysely<Database>, input: CreateSessionIn
.execute();
} catch (error) {
if (isUniqueViolation(error)) {
if (isPgConstraint(error, 'session_external_id_uq') && input.external_id) {
if (isPgConstraint(error, SESSION_EXTERNAL_ID_UQ) && input.external_id) {
throw new SessionExternalIdConflictError(input.external_id, { cause: error });
}
throw new SessionAlreadyExistsError(input.session_id, { cause: error });
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { type Kysely, sql } from 'kysely';
import { SESSION_EXTERNAL_ID_UQ } from '../../indexes';

/**
* Optional session `external_id`, unique within a tenant when set.
Expand All @@ -8,7 +9,7 @@ export async function up(db: Kysely<unknown>): Promise<void> {
await db.transaction().execute(async trx => {
await sql`ALTER TABLE session ADD COLUMN external_id TEXT`.execute(trx);
await sql`
CREATE UNIQUE INDEX ${sql.raw('session_external_id_uq')}
CREATE UNIQUE INDEX ${sql.raw(SESSION_EXTERNAL_ID_UQ)}
ON session (tenant_id, external_id)
WHERE external_id IS NOT NULL
`.execute(trx);
Expand All @@ -17,7 +18,7 @@ export async function up(db: Kysely<unknown>): Promise<void> {

export async function down(db: Kysely<unknown>): Promise<void> {
await db.transaction().execute(async trx => {
await sql`DROP INDEX IF EXISTS ${sql.raw('session_external_id_uq')}`.execute(trx);
await sql`DROP INDEX IF EXISTS ${sql.raw(SESSION_EXTERNAL_ID_UQ)}`.execute(trx);
await sql`ALTER TABLE session DROP COLUMN external_id`.execute(trx);
});
}
20 changes: 7 additions & 13 deletions packages/trueforge/src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ import { ModelCatalog } from './catalog/ModelCatalog';
import { SandboxCatalog } from './catalog/SandboxCatalog';
import { SkillCatalog } from './catalog/SkillCatalog';
import { type DistributedServerConfiguration } from './config';
import { Controller } from './controller/Controller';
import { createScheduleDispatchLoop, logTriggeredRun } from './controller/scheduleDispatch';
import { createController } from './controller';
import type { IAgentStore } from './db/agentStore';
import type { IMcpServerStore } from './db/mcpServerStore';
import type { IModelProviderStore } from './db/modelProviderStore';
Expand Down Expand Up @@ -233,16 +232,11 @@ async function createServerRuntime<TTransaction>(persistence: ServerPersistence<

// Standalone is one process, so it owns the control loops too.
const controller = configuration.STANDALONE
? new Controller({
loops: [
createScheduleDispatchLoop({
scheduleStore,
withTransaction,
onTriggered: logTriggeredRun(logger),
logger,
}),
],
? createController({
scheduleStore,
withTransaction,
logger,
baseUrl: `http://localhost:${String(configuration.PORT)}`,
})
: undefined;

Expand All @@ -269,8 +263,6 @@ async function createServerRuntime<TTransaction>(persistence: ServerPersistence<
oidcClient,
});

controller?.start();

return { activeTurns, app, controller, destroyDb, redis, requestReplyRouter };
}

Expand Down Expand Up @@ -353,6 +345,8 @@ try {

const server = serve({ fetch: app.fetch, port: configuration.PORT, hostname: configuration.HOST }, info => {
logger.info(`Agent server listening on http://${configuration.HOST}:${String(info.port)} (docs at /api/v1/docs)`);
// The controller calls this server over HTTP.
controller?.start();
});

server.on('error', (error: unknown) => {
Expand Down
Loading