Skip to content

Commit 23eb4ec

Browse files
Update session metrics on createTurn and terminal writes
1 parent 27c49cf commit 23eb4ec

6 files changed

Lines changed: 277 additions & 13 deletions

File tree

‎.changeset/20260827210200-session-metrics.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
'@truefoundry/trueforge': patch
44
---
55

6-
Persist zero-initialized metrics on agent sessions.
6+
Persist zero-initialized session metrics and fold totals on createTurn and terminal writes.

‎packages/trueforge-core/src/agent-session/store/ISessionStore.ts‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,8 @@ export interface ISessionStore<
262262
* row or leave `last_turn_id` pointing at a turn that was never created.
263263
* The implementation supplies the mechanism (session lock, row lock/tx, …).
264264
*
265-
* Also bumps `session.last_activity_timestamp_ms` in that same atomic unit.
265+
* Also bumps `session.last_activity_timestamp_ms` and increments
266+
* `session.metrics.total_turns` in that same atomic unit.
266267
*
267268
* Fork semantics for `turn.previous_turn_id`:
268269
* - `null` — new root turn (no parent); always allowed.
@@ -282,9 +283,10 @@ export interface ISessionStore<
282283

283284
/**
284285
* One tx: (a) conditionally cancel a running turn with `reason`; (b) if it
285-
* did cancel, insert the caller-built `turn_done_event` — already-terminal
286-
* turns skip the event insert; (c) return the now-immutable turn record.
287-
* Missing turn → {@link TurnNotFoundError}.
286+
* did cancel, insert the caller-built `turn_done_event` and add cost plus
287+
* `completed_at − created_at` into `session.metrics` — already-terminal
288+
* turns skip the event insert and the metrics fold; (c) return the now-immutable
289+
* turn record. Missing turn → {@link TurnNotFoundError}.
288290
*/
289291
freezeAndGetTurn(input: FreezeAndGetTurnInput): Promise<TurnRecord<TTurnCustom>>;
290292

@@ -297,7 +299,9 @@ export interface ISessionStore<
297299
): Promise<{ data: TurnRecordWithoutSnapshot<TTurnCustom>[]; pagination: TokenPagination }>;
298300

299301
/**
300-
* Writes the terminal state and `turn_done_event` atomically. Store contract —
302+
* Writes the terminal state and `turn_done_event` atomically, and adds
303+
* `metrics.total_cost_in_usd` plus `completed_at − created_at` into
304+
* `session.metrics` in that same unit. Store contract —
301305
* **first terminal write wins**:
302306
* - Allowed: `running` → `done` | `cancelled` | `error`.
303307
* - Rejected with **409** (or equivalent conflict): any write when status is already

‎packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,7 @@ export class InMemorySessionStore<
337337
this.events.set(tKey, []);
338338
stored.turnIds.push(input.turn.turn_id);
339339
stored.record.last_turn_id = input.turn.turn_id;
340+
stored.record.metrics.total_turns += 1;
340341
stored.record.last_activity_timestamp_ms = Date.now();
341342
stored.record.updated_at = new Date();
342343
if (input.update_session_title_if_not_exist !== null && stored.record.title === null) {
@@ -360,6 +361,7 @@ export class InMemorySessionStore<
360361
if (list) {
361362
list.push(deepCopy(input.turn_done_event));
362363
}
364+
this.addTerminalSessionMetrics(input.session_id, turn.created_at, cancelledState);
363365
}
364366

365367
return deepCopy(turn);
@@ -402,6 +404,7 @@ export class InMemorySessionStore<
402404
if (list) {
403405
list.push(deepCopy(input.turn_done_event));
404406
}
407+
this.addTerminalSessionMetrics(input.session_id, turn.created_at, input.state);
405408
}
406409

407410
async appendToEvents(input: AppendToEventsInput): Promise<void> {
@@ -415,6 +418,17 @@ export class InMemorySessionStore<
415418
return;
416419
}
417420

421+
/** Cost from turn metrics; duration is completed_at − created_at, floored at 0. */
422+
private addTerminalSessionMetrics(sessionId: string, created_at: Date, state: TerminalTurnState): void {
423+
const stored = this.sessions.get(sessionKey(sessionId));
424+
if (!stored) {
425+
throw new SessionNotFoundError(sessionId);
426+
}
427+
const elapsed_ms = Date.parse(state.completed_at) - created_at.getTime();
428+
stored.record.metrics.total_cost_in_usd += state.metrics?.total_cost_in_usd ?? 0;
429+
stored.record.metrics.total_duration_ms += elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;
430+
}
431+
418432
private requireTurn(sessionId: string, turnId: string): TurnRecord<TTurnCustom> {
419433
const turn = this.turns.get(turnKey({ session_id: sessionId, turn_id: turnId }));
420434
if (!turn) {

‎packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -840,6 +840,24 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
840840
expect(mustGet(after).last_activity_timestamp_ms).toBeGreaterThan(mustGet(before).last_activity_timestamp_ms);
841841
});
842842

843+
it('increments session.metrics.total_turns without cost or duration', async () => {
844+
const store = createStore();
845+
await seedSession(store);
846+
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
847+
const afterFirst = await store.getSession({ tenant_id: tenant, session_id: sessionId });
848+
expect(mustGet(afterFirst).metrics).toEqual({
849+
total_cost_in_usd: 0,
850+
total_duration_ms: 0,
851+
total_turns: 1,
852+
});
853+
await finishTurn(store, 'turn-1');
854+
await store.createTurn(
855+
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
856+
);
857+
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
858+
expect(mustGet(afterSecond).metrics.total_turns).toBe(2);
859+
});
860+
843861
it('update_session_title_if_not_exist sets once and never overwrites', async () => {
844862
const store = createStore();
845863
await seedSession(store);
@@ -1322,6 +1340,38 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
13221340
});
13231341
});
13241342

1343+
it('folds duration into session.metrics when cancel applies, not on a second freeze', async () => {
1344+
const store = createStore();
1345+
await seedSession(store);
1346+
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
1347+
const cancelledState = makeCancelledTurnState(CancellationReason.CancelledForNextTurn);
1348+
const record = await store.freezeAndGetTurn({
1349+
session_id: sessionId,
1350+
turn_id: 'turn-1',
1351+
reason: CancellationReason.CancelledForNextTurn,
1352+
turn_done_event: makeTurnDoneEvent(cancelledState),
1353+
});
1354+
if (record.state.status !== 'cancelled') {
1355+
throw new Error(`expected cancelled turn, got ${record.state.status}`);
1356+
}
1357+
const afterCancel = await store.getSession({ tenant_id: tenant, session_id: sessionId });
1358+
const elapsed_ms = Date.parse(record.state.completed_at) - record.created_at.getTime();
1359+
expect(mustGet(afterCancel).metrics).toEqual({
1360+
total_cost_in_usd: 0,
1361+
total_duration_ms: elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0,
1362+
total_turns: 1,
1363+
});
1364+
1365+
await store.freezeAndGetTurn({
1366+
session_id: sessionId,
1367+
turn_id: 'turn-1',
1368+
reason: CancellationReason.CancelledForNextTurn,
1369+
turn_done_event: makeTurnDoneEvent(cancelledState),
1370+
});
1371+
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
1372+
expect(mustGet(afterSecond).metrics).toEqual(mustGet(afterCancel).metrics);
1373+
});
1374+
13251375
it('on an already-terminal turn is a plain read without duplicating turn.done', async () => {
13261376
const store = createStore();
13271377
await seedSession(store);
@@ -1460,6 +1510,115 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
14601510
expect(doneEvents[0]).toEqual(turnDone);
14611511
});
14621512

1513+
it('adds cost and duration into session.metrics on running → terminal', async () => {
1514+
const store = createStore();
1515+
await seedSession(store);
1516+
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
1517+
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
1518+
const createdAt = mustGet(turn).created_at;
1519+
const completedAt = new Date(createdAt.getTime() + 1500).toISOString();
1520+
const state = {
1521+
...makeDoneTurnState(),
1522+
completed_at: completedAt,
1523+
metrics: { total_cost_in_usd: 1.25 },
1524+
};
1525+
await store.updateTurnState({
1526+
session_id: sessionId,
1527+
turn_id: 'turn-1',
1528+
state,
1529+
turn_done_event: makeTurnDoneEvent(state),
1530+
});
1531+
const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
1532+
expect(mustGet(session).metrics).toEqual({
1533+
total_cost_in_usd: 1.25,
1534+
total_duration_ms: 1500,
1535+
total_turns: 1,
1536+
});
1537+
});
1538+
1539+
it('does not add session.metrics again on a losing terminal write', async () => {
1540+
const store = createStore();
1541+
await seedSession(store);
1542+
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
1543+
const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
1544+
const createdAt = mustGet(turn).created_at;
1545+
const doneState = {
1546+
...makeDoneTurnState(),
1547+
completed_at: new Date(createdAt.getTime() + 1500).toISOString(),
1548+
metrics: { total_cost_in_usd: 1.25 },
1549+
};
1550+
await store.updateTurnState({
1551+
session_id: sessionId,
1552+
turn_id: 'turn-1',
1553+
state: doneState,
1554+
turn_done_event: makeTurnDoneEvent(doneState),
1555+
});
1556+
const afterFirst = mustGet(await store.getSession({ tenant_id: tenant, session_id: sessionId })).metrics;
1557+
1558+
const losingState = {
1559+
...makeCancelledTurnState(CancellationReason.ClientCancelled),
1560+
completed_at: new Date(createdAt.getTime() + 8000).toISOString(),
1561+
metrics: { total_cost_in_usd: 9.99 },
1562+
};
1563+
await expect(
1564+
store.updateTurnState({
1565+
session_id: sessionId,
1566+
turn_id: 'turn-1',
1567+
state: losingState,
1568+
turn_done_event: makeTurnDoneEvent(losingState),
1569+
}),
1570+
).rejects.toBeInstanceOf(SessionStoreConflictError);
1571+
1572+
const afterSecond = await store.getSession({ tenant_id: tenant, session_id: sessionId });
1573+
expect(mustGet(afterSecond).metrics).toEqual(afterFirst);
1574+
expect(afterFirst).toEqual({
1575+
total_cost_in_usd: 1.25,
1576+
total_duration_ms: 1500,
1577+
total_turns: 1,
1578+
});
1579+
});
1580+
1581+
it('adds cost and duration from a second done turn onto existing session.metrics', async () => {
1582+
const store = createStore();
1583+
await seedSession(store);
1584+
await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));
1585+
const turn1 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });
1586+
const turn1Done = {
1587+
...makeDoneTurnState(),
1588+
completed_at: new Date(mustGet(turn1).created_at.getTime() + 1500).toISOString(),
1589+
metrics: { total_cost_in_usd: 1.25 },
1590+
};
1591+
await store.updateTurnState({
1592+
session_id: sessionId,
1593+
turn_id: 'turn-1',
1594+
state: turn1Done,
1595+
turn_done_event: makeTurnDoneEvent(turn1Done),
1596+
});
1597+
1598+
await store.createTurn(
1599+
makeCreateTurnInput({ sessionId, turnId: 'turn-2', previousTurnId: 'turn-1', firstTurnId: 'turn-1' }),
1600+
);
1601+
const turn2 = await store.getTurn({ session_id: sessionId, turn_id: 'turn-2' });
1602+
const turn2Done = {
1603+
...makeDoneTurnState(),
1604+
completed_at: new Date(mustGet(turn2).created_at.getTime() + 800).toISOString(),
1605+
metrics: { total_cost_in_usd: 0.5 },
1606+
};
1607+
await store.updateTurnState({
1608+
session_id: sessionId,
1609+
turn_id: 'turn-2',
1610+
state: turn2Done,
1611+
turn_done_event: makeTurnDoneEvent(turn2Done),
1612+
});
1613+
1614+
const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });
1615+
expect(mustGet(session).metrics).toEqual({
1616+
total_cost_in_usd: 1.75,
1617+
total_duration_ms: 2300,
1618+
total_turns: 2,
1619+
});
1620+
});
1621+
14631622
it('missing turn → not found', async () => {
14641623
const store = createStore();
14651624
await seedSession(store);

‎packages/trueforge/src/db/postgres/session-store/queries/turns.ts‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import type { SessionMetrics } from '@truefoundry/trueforge-core/agent-session';
12
import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';
23
import {
34
type TerminalTurnState,
@@ -28,7 +29,7 @@ import { getEmptyCurrentContextUsage } from '@truefoundry/trueforge-core/core/ru
2829
import type { SandboxInfo } from '@truefoundry/trueforge-core/core/sandbox/Sandbox';
2930
import { sql, type Kysely, type QueryCreator, type RawBuilder, type Transaction } from 'kysely';
3031
import { isUniqueViolation } from '../../client';
31-
import { json } from '../../sqlExpressions';
32+
import { json, jsonbSet } from '../../sqlExpressions';
3233
import type { Database, TurnCheckpoint, TurnThreadCheckpoint } from '../../types';
3334
import { lateralUnnestBigintArrayWithOrdinality } from '../sqlExpressions';
3435

@@ -110,6 +111,39 @@ export interface ListTurnsResult {
110111
type DbOrTrx = Kysely<Database> | Transaction<Database>;
111112
type TurnFenceDb = DbOrTrx | QueryCreator<Database>;
112113

114+
function incrementSessionTotalTurns(): RawBuilder<SessionMetrics> {
115+
return jsonbSet<SessionMetrics>(
116+
sql`metrics`,
117+
sql`'{total_turns}'`,
118+
sql`to_jsonb((metrics->>'total_turns')::int + 1)`,
119+
);
120+
}
121+
122+
/** Same tx as the turn flip; the session row lock serializes concurrent terminal folds. */
123+
async function addSessionCostAndDuration(
124+
trx: Transaction<Database>,
125+
input: { session_id: string; created_at: Date; state: TerminalTurnState },
126+
): Promise<void> {
127+
const elapsed_ms = Date.parse(input.state.completed_at) - input.created_at.getTime();
128+
const total_cost_in_usd = input.state.metrics?.total_cost_in_usd ?? 0;
129+
const total_duration_ms = elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;
130+
await trx
131+
.updateTable('session')
132+
.set({
133+
metrics: jsonbSet<SessionMetrics>(
134+
jsonbSet(
135+
sql`metrics`,
136+
sql`'{total_cost_in_usd}'`,
137+
sql`to_jsonb((metrics->>'total_cost_in_usd')::double precision + ${total_cost_in_usd}::double precision)`,
138+
),
139+
sql`'{total_duration_ms}'`,
140+
sql`to_jsonb((metrics->>'total_duration_ms')::int + ${total_duration_ms}::int)`,
141+
),
142+
})
143+
.where('session_id', '=', input.session_id)
144+
.execute();
145+
}
146+
113147
function terminalTurnState(state: TurnState, turn_id: string): TerminalTurnState {
114148
switch (state.status) {
115149
case 'running':
@@ -384,6 +418,8 @@ export async function createTurn(db: Kysely<Database>, input: CreateTurnInput):
384418
last_turn_id: input.turn.turn_id,
385419
updated_at: sql`now()`,
386420
last_activity_timestamp_ms: input.last_activity_timestamp_ms,
421+
// total_turns rides the same tip UPDATE so a later failure in this tx rolls it back.
422+
metrics: incrementSessionTotalTurns(),
387423
...(input.update_session_title_if_not_exist !== null
388424
? {
389425
title: sql`COALESCE(title, ${input.update_session_title_if_not_exist})`,
@@ -618,9 +654,10 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
618654
.where('session_id', '=', input.session_id)
619655
.where('turn_id', '=', input.turn_id)
620656
.where(sql<boolean>`state->>'status' = 'running'`)
657+
.returning(['created_at'])
621658
.executeTakeFirst();
622659

623-
if (Number(updateResult.numUpdatedRows) > 0) {
660+
if (updateResult !== undefined) {
624661
await trx
625662
.insertInto('session_event')
626663
.values({
@@ -631,6 +668,12 @@ export async function freezeAndGetTurn(db: Kysely<Database>, input: FreezeAndGet
631668
created_at: new Date(input.turn_done_event.created_at),
632669
})
633670
.execute();
671+
// Only the winning cancel folds; a freeze of an already-terminal turn is a read.
672+
await addSessionCostAndDuration(trx, {
673+
session_id: input.session_id,
674+
created_at: updateResult.created_at,
675+
state: cancelledState,
676+
});
634677
}
635678

636679
const record = await assembleTurnRecord(trx, input);
@@ -706,10 +749,11 @@ export async function updateTurnState(db: Kysely<Database>, input: UpdateTurnSta
706749
.where('session_id', '=', input.session_id)
707750
.where('turn_id', '=', input.turn_id)
708751
.where(sql<boolean>`state->>'status' = 'running'`)
752+
.returning(['created_at'])
709753
.executeTakeFirst();
710754

711-
const numUpdated = Number(result.numUpdatedRows);
712-
if (numUpdated === 0) {
755+
// No RETURNING row: UPDATE matched 0 running turns.
756+
if (result === undefined) {
713757
const existing = await trx
714758
.selectFrom('turn')
715759
.select('state')
@@ -723,6 +767,12 @@ export async function updateTurnState(db: Kysely<Database>, input: UpdateTurnSta
723767
throw new TurnNotRunningError(input.turn_id, terminalTurnState(existing.state, input.turn_id));
724768
}
725769

770+
await addSessionCostAndDuration(trx, {
771+
session_id: input.session_id,
772+
created_at: result.created_at,
773+
state: input.state,
774+
});
775+
726776
await trx
727777
.insertInto('session_event')
728778
.values({

0 commit comments

Comments
 (0)