Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changeset/20260831083812-session-metrics-fold.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
'@truefoundry/trueforge-core': patch
'@truefoundry/trueforge': patch
---

Fold session metrics totals on createTurn and terminal writes.
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,8 @@ export interface ISessionStore<
* row or leave `last_turn_id` pointing at a turn that was never created.
* The implementation supplies the mechanism (session lock, row lock/tx, …).
*
* Also bumps `session.last_activity_timestamp_ms` in that same atomic unit.
* Also bumps `session.last_activity_timestamp_ms` and increments
* `session.metrics.total_turns` in that same atomic unit.
*
* Fork semantics for `turn.previous_turn_id`:
* - `null` — new root turn (no parent); always allowed.
Expand All @@ -282,9 +283,10 @@ export interface ISessionStore<

/**
* One tx: (a) conditionally cancel a running turn with `reason`; (b) if it
* did cancel, insert the caller-built `turn_done_event` — already-terminal
* turns skip the event insert; (c) return the now-immutable turn record.
* Missing turn → {@link TurnNotFoundError}.
* did cancel, insert the caller-built `turn_done_event` and add cost plus
* `completed_at − created_at` into `session.metrics` — already-terminal
Comment thread
sr07asthana marked this conversation as resolved.
Resolved
* turns skip the event insert and the metrics fold; (c) return the now-immutable
* turn record. Missing turn → {@link TurnNotFoundError}.
*/
freezeAndGetTurn(input: FreezeAndGetTurnInput): Promise<TurnRecord<TTurnCustom>>;

Expand All @@ -297,7 +299,9 @@ export interface ISessionStore<
): Promise<{ data: TurnRecordWithoutSnapshot<TTurnCustom>[]; pagination: TokenPagination }>;

/**
* Writes the terminal state and `turn_done_event` atomically. Store contract —
* Writes the terminal state and `turn_done_event` atomically, and adds
* `metrics.total_cost_in_usd` plus `completed_at − created_at` into
* `session.metrics` in that same unit. Store contract —
* **first terminal write wins**:
* - Allowed: `running` → `done` | `cancelled` | `error`.
* - Rejected with **409** (or equivalent conflict): any write when status is already
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,7 @@ export class InMemorySessionStore<
this.events.set(tKey, []);
stored.turnIds.push(input.turn.turn_id);
stored.record.last_turn_id = input.turn.turn_id;
stored.record.metrics.total_turns += 1;
stored.record.last_activity_timestamp_ms = Date.now();
stored.record.updated_at = new Date();
if (input.update_session_title_if_not_exist !== null && stored.record.title === null) {
Expand All @@ -360,6 +361,7 @@ export class InMemorySessionStore<
if (list) {
list.push(deepCopy(input.turn_done_event));
}
this.addTerminalSessionMetrics(input.session_id, turn.created_at, cancelledState);
}

return deepCopy(turn);
Expand Down Expand Up @@ -402,6 +404,7 @@ export class InMemorySessionStore<
if (list) {
list.push(deepCopy(input.turn_done_event));
}
this.addTerminalSessionMetrics(input.session_id, turn.created_at, input.state);
}

async appendToEvents(input: AppendToEventsInput): Promise<void> {
Expand All @@ -415,6 +418,17 @@ export class InMemorySessionStore<
return;
}

/** Cost from turn metrics; duration is completed_at − created_at, floored at 0. */
private addTerminalSessionMetrics(sessionId: string, created_at: Date, state: TerminalTurnState): void {
const stored = this.sessions.get(sessionKey(sessionId));
if (!stored) {
throw new SessionNotFoundError(sessionId);
}
const elapsed_ms = Date.parse(state.completed_at) - created_at.getTime();
stored.record.metrics.total_cost_in_usd += state.metrics?.total_cost_in_usd ?? 0;
Comment thread
sr07asthana marked this conversation as resolved.
Resolved
stored.record.metrics.total_duration_ms += elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;
}

private requireTurn(sessionId: string, turnId: string): TurnRecord<TTurnCustom> {
const turn = this.turns.get(turnKey({ session_id: sessionId, turn_id: turnId }));
if (!turn) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,24 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
expect(mustGet(after).last_activity_timestamp_ms).toBeGreaterThan(mustGet(before).last_activity_timestamp_ms);
});

it('increments session.metrics.total_turns without cost or duration', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const afterFirst = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterFirst).metrics).toEqual({
total_cost_in_usd: 0,
total_duration_ms: 0,
total_turns: 1,
});
await finishTurn(store, 'turn-1');
await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
);
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics.total_turns).toBe(2);
});

it('update_session_title_if_not_exist sets once and never overwrites', async () => {
const store = createStore();
await seedSession(store);
Expand Down Expand Up @@ -1322,6 +1340,38 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
});
});

it('folds duration into session.metrics when cancel applies, not on a second freeze', async () => {
Comment thread
sr07asthana marked this conversation as resolved.
Resolved
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const cancelledState = makeCancelledTurnState(CancellationReason.CancelledForNextTurn);
const record = await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
if (record.state.status !== 'cancelled') {
throw new Error(`expected cancelled turn, got ${record.state.status}`);
}
const afterCancel = await store.getSession({ tenant_id: tenant, session_id: sessionId });
const elapsed_ms = Date.parse(record.state.completed_at) - record.created_at.getTime();
expect(mustGet(afterCancel).metrics).toEqual({
total_cost_in_usd: 0,
total_duration_ms: elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0,
total_turns: 1,
});

await store.freezeAndGetTurn({
session_id: sessionId,
turn_id: 'turn-1',
reason: CancellationReason.CancelledForNextTurn,
turn_done_event: makeTurnDoneEvent(cancelledState),
});
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics).toEqual(mustGet(afterCancel).metrics);
});

it('on an already-terminal turn is a plain read without duplicating turn.done', async () => {
const store = createStore();
await seedSession(store);
Expand Down Expand Up @@ -1460,6 +1510,115 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
expect(doneEvents[0]).toEqual(turnDone);
});

it('adds cost and duration into session.metrics on running → terminal', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const createdAt = mustGet(turn).created_at;
const completedAt = new Date(createdAt.getTime() + 1500).toISOString();
const state = {
...makeDoneTurnState(),
completed_at: completedAt,
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state,
turn_done_event: makeTurnDoneEvent(state),
});
const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(session).metrics).toEqual({
total_cost_in_usd: 1.25,
total_duration_ms: 1500,
total_turns: 1,
});
});

it('does not add session.metrics again on a losing terminal write', async () => {
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const createdAt = mustGet(turn).created_at;
const doneState = {
...makeDoneTurnState(),
completed_at: new Date(createdAt.getTime() + 1500).toISOString(),
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: doneState,
turn_done_event: makeTurnDoneEvent(doneState),
});
const afterFirst = mustGet(await store.getSession({ tenant_id: tenant, session_id: sessionId })).metrics;

const losingState = {
...makeCancelledTurnState(CancellationReason.ClientCancelled),
completed_at: new Date(createdAt.getTime() + 8000).toISOString(),
metrics: { total_cost_in_usd: 9.99 },
};
await expect(
store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: losingState,
turn_done_event: makeTurnDoneEvent(losingState),
}),
).rejects.toBeInstanceOf(SessionStoreConflictError);

const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(afterSecond).metrics).toEqual(afterFirst);
expect(afterFirst).toEqual({
total_cost_in_usd: 1.25,
total_duration_ms: 1500,
total_turns: 1,
});
});

it('adds cost and duration from a second done turn onto existing session.metrics', async () => {
Comment thread
sr07asthana marked this conversation as resolved.
Resolved
const store = createStore();
await seedSession(store);
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
const turn1 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
const turn1Done = {
...makeDoneTurnState(),
completed_at: new Date(mustGet(turn1).created_at.getTime() + 1500).toISOString(),
metrics: { total_cost_in_usd: 1.25 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-1',
state: turn1Done,
turn_done_event: makeTurnDoneEvent(turn1Done),
});

await store.createTurn(
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
);
const turn2 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-2' });
const turn2Done = {
...makeDoneTurnState(),
completed_at: new Date(mustGet(turn2).created_at.getTime() + 800).toISOString(),
metrics: { total_cost_in_usd: 0.5 },
};
await store.updateTurnState({
session_id: sessionId,
turn_id: 'turn-2',
state: turn2Done,
turn_done_event: makeTurnDoneEvent(turn2Done),
});

const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
expect(mustGet(session).metrics).toEqual({
total_cost_in_usd: 1.75,
total_duration_ms: 2300,
total_turns: 2,
});
});

it('missing turn → not found', async () => {
const store = createStore();
await seedSession(store);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import type { SessionMetrics } from '@truefoundry/trueforge-core/agent-session';
import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';
import {
type TerminalTurnState,
Expand Down Expand Up @@ -28,7 +29,7 @@ import { getEmptyCurrentContextUsage } from '@truefoundry/trueforge-core/core/ru
import type { SandboxInfo } from '@truefoundry/trueforge-core/core/sandbox/Sandbox';
import { sql, type Kysely, type QueryCreator, type RawBuilder, type Transaction } from 'kysely';
import { isUniqueViolation } from '../../client';
import { json } from '../../sqlExpressions';
import { json, jsonbSet } from '../../sqlExpressions';
import type { Database, TurnCheckpoint, TurnThreadCheckpoint } from '../../types';
import { lateralUnnestBigintArrayWithOrdinality } from '../sqlExpressions';

Expand Down Expand Up @@ -110,6 +111,40 @@ export interface ListTurnsResult {
type DbOrTrx = Kysely<Database> | Transaction<Database>;
type TurnFenceDb = DbOrTrx | QueryCreator<Database>;

function incrementSessionTotalTurns(): RawBuilder<SessionMetrics> {
return jsonbSet<SessionMetrics>(
sql`metrics`,
sql`'{total_turns}'`,
sql`to_jsonb((metrics->>'total_turns')::int + 1)`,
);
}

/** Same tx as the turn flip; the session row lock serializes concurrent terminal folds. */
async function addSessionCostAndDuration(
trx: Transaction<Database>,
input: { session_id: string; created_at: Date; state: TerminalTurnState },
): Promise<void> {
const elapsed_ms = Date.parse(input.state.completed_at) - input.created_at.getTime();
const total_cost_in_usd = input.state.metrics?.total_cost_in_usd ?? 0;
const total_duration_ms = elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;
await trx
.updateTable('session')
.set({
metrics: jsonbSet<SessionMetrics>(
jsonbSet(
sql`metrics`,
sql`'{total_cost_in_usd}'`,
sql`to_jsonb((metrics->>'total_cost_in_usd')::double precision + ${total_cost_in_usd}::double precision)`,
),
// bigint: ::int overflows at ~24.8 days of summed ms and would roll back the terminal tx.
sql`'{total_duration_ms}'`,
sql`to_jsonb((metrics->>'total_duration_ms')::bigint + ${total_duration_ms}::bigint)`,
),
})
.where('session_id', '=', input.session_id)
.execute();
}

function terminalTurnState(state: TurnState, turn_id: string): TerminalTurnState {
switch (state.status) {
case 'running':
Expand Down Expand Up @@ -384,6 +419,8 @@ export async function createTurn(db: Kysely<Database>, input: CreateTurnInput):
last_turn_id: input.turn.turn_id,
updated_at: sql`now()`,
last_activity_timestamp_ms: input.last_activity_timestamp_ms,
// total_turns rides the same tip UPDATE so a later failure in this tx rolls it back.
metrics: incrementSessionTotalTurns(),
...(input.update_session_title_if_not_exist !== null
? {
title: sql`COALESCE(title, ${input.update_session_title_if_not_exist})`,
Expand Down Expand Up @@ -618,9 +655,10 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
.where('session_id', '=', input.session_id)
.where('turn_id', '=', input.turn_id)
.where(sql<boolean>`state->>'status' = 'running'`)
.returning(['created_at'])
.executeTakeFirst();

if (Number(updateResult.numUpdatedRows) > 0) {
if (updateResult !== undefined) {
await trx
.insertInto('session_event')
.values({
Expand All @@ -631,6 +669,12 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
created_at: new Date(input.turn_done_event.created_at),
})
.execute();
// Only the winning cancel folds; a freeze of an already-terminal turn is a read.
await addSessionCostAndDuration(trx, {
session_id: input.session_id,
created_at: updateResult.created_at,
state: cancelledState,
});
}

const record = await assembleTurnRecord(trx, input);
Expand Down Expand Up @@ -706,10 +750,11 @@ export async function updateTurnState(db: Kysely<Database>, input: UpdateTurnSta
.where('session_id', '=', input.session_id)
.where('turn_id', '=', input.turn_id)
.where(sql<boolean>`state->>'status' = 'running'`)
.returning(['created_at'])
.executeTakeFirst();

const numUpdated = Number(result.numUpdatedRows);
if (numUpdated === 0) {
// No RETURNING row: UPDATE matched 0 running turns.
if (result === undefined) {
const existing = await trx
.selectFrom('turn')
.select('state')
Expand All @@ -723,6 +768,12 @@ export async function updateTurnState(db: Kysely<Database>, input: UpdateTurnSta
throw new TurnNotRunningError(input.turn_id, terminalTurnState(existing.state, input.turn_id));
}

await addSessionCostAndDuration(trx, {
session_id: input.session_id,
created_at: result.created_at,
state: input.state,
});

await trx
.insertInto('session_event')
.values({
Expand Down
Loading