Skip to content

Commit 4f7e36f

Browse files
Add session metrics API and per-session totals
1 parent 5f4ff3a commit 4f7e36f

17 files changed

Lines changed: 1416 additions & 7 deletions

File tree

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
'@truefoundry/trueforge-core': patch
3+
'@truefoundry/trueforge': patch
4+
---
5+
6+
Add caller-scoped session metrics meters, chart catalog, and chart-data endpoints for named agents.

‎packages/trueforge-core/src/agent-session/index.ts‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,35 @@ export {
2121
} from './schemas/turn';
2222
export type { TerminalTurnState, Turn, TurnInputItem, TurnMetrics, TurnState } from './schemas/turn';
2323

24-
export { SessionMetricsSchema, SessionSchema } from './schemas/session';
25-
export type { Session, SessionAgent, SessionMetrics } from './schemas/session';
24+
export {
25+
SessionMetricsChartNameSchema,
26+
SessionMetricsChartSchema,
27+
SessionMetricsChartsDataResponseSchema,
28+
SessionMetricsChartsResponseSchema,
29+
SessionMetricsGraphLineSchema,
30+
SessionMetricsGraphSchema,
31+
SessionMetricsMeterNameSchema,
32+
SessionMetricsMeterSchema,
33+
SessionMetricsMetersResponseSchema,
34+
SessionMetricsPointSchema,
35+
SessionMetricsSchema,
36+
SessionSchema,
37+
} from './schemas/session';
38+
export type {
39+
Session,
40+
SessionAgent,
41+
SessionMetrics,
42+
SessionMetricsChart,
43+
SessionMetricsChartName,
44+
SessionMetricsChartsDataResponse,
45+
SessionMetricsChartsResponse,
46+
SessionMetricsGraph,
47+
SessionMetricsGraphLine,
48+
SessionMetricsMeter,
49+
SessionMetricsMeterName,
50+
SessionMetricsMetersResponse,
51+
SessionMetricsPoint,
52+
} from './schemas/session';
2653

2754
export {
2855
EventType,
@@ -57,6 +84,8 @@ export type {
5784
DeleteSessionInput,
5885
FreezeAndGetTurnInput,
5986
GetSessionInput,
87+
GetSessionMetricsChartDataInput,
88+
GetSessionMetricsInput,
6089
GetTurnInput,
6190
ISessionStore,
6291
ListSessionEventsInput,

‎packages/trueforge-core/src/agent-session/schemas/session.ts‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,100 @@ export const SessionMetricsSchema = z
1111
total_duration_ms: z.number().int().nonnegative(),
1212
total_turns: z.number().int().nonnegative(),
1313
})
14-
.strict();
14+
.strict()
15+
.describe('Rolled-up cost, duration, and turn counters for a session.')
16+
.openapi('SessionMetrics');
17+
18+
export const SessionMetricsPointSchema = z
19+
.object({
20+
timestamp: z.string(),
21+
value: z.number().nonnegative(),
22+
})
23+
.strict()
24+
.openapi('SessionMetricsPoint');
25+
26+
export const SessionMetricsMeterNameSchema = z.enum([
27+
'total_sessions',
28+
'total_cost_in_usd',
29+
'total_turns',
30+
'cost_per_session_in_usd',
31+
'avg_turns_per_session',
32+
'min_turns_per_session',
33+
'max_turns_per_session',
34+
'median_turns_per_session',
35+
'min_session_duration_ms',
36+
'max_session_duration_ms',
37+
'median_session_duration_ms',
38+
'p95_session_duration_ms',
39+
]);
40+
41+
export const SessionMetricsMeterSchema = z
42+
.object({
43+
name: SessionMetricsMeterNameSchema,
44+
aggregate_value: z.number().nonnegative(),
45+
description: z.string(),
46+
unit: z.enum(['count', '$', 'ms']),
47+
})
48+
.strict()
49+
.openapi('SessionMetricsMeter');
50+
51+
export const SessionMetricsMetersResponseSchema = z
52+
.object({
53+
meters: z.array(SessionMetricsMeterSchema).length(12),
54+
})
55+
.strict()
56+
.openapi('SessionMetricsMetersResponse');
57+
58+
export const SessionMetricsChartNameSchema = z.enum([
59+
'sessions_over_time',
60+
'sessions_cost_over_time',
61+
'turns_over_time',
62+
]);
63+
64+
export const SessionMetricsChartSchema = z
65+
.object({
66+
name: SessionMetricsChartNameSchema,
67+
display_name: z.string(),
68+
description: z.string(),
69+
chart_type: z.literal('line'),
70+
})
71+
.strict()
72+
.openapi('SessionMetricsChart');
73+
74+
export const SessionMetricsChartsResponseSchema = z
75+
.object({
76+
charts: z.array(SessionMetricsChartSchema).length(3),
77+
})
78+
.strict()
79+
.openapi('SessionMetricsChartsResponse');
80+
81+
export const SessionMetricsGraphLineSchema = z
82+
.object({
83+
name: z.string(),
84+
values: z.array(SessionMetricsPointSchema),
85+
})
86+
.strict()
87+
.openapi('SessionMetricsGraphLine');
88+
89+
export const SessionMetricsGraphSchema = z
90+
.object({
91+
name: SessionMetricsChartNameSchema,
92+
display_name: z.string(),
93+
description: z.string(),
94+
unit: z.enum(['count', '$']),
95+
chart_type: z.literal('line'),
96+
graph_lines: z.array(SessionMetricsGraphLineSchema).length(1),
97+
})
98+
.strict()
99+
.openapi('SessionMetricsGraph');
100+
101+
export const SessionMetricsChartsDataResponseSchema = z
102+
.object({
103+
step: z.string(),
104+
graphs: z.array(SessionMetricsGraphSchema).length(1),
105+
})
106+
.strict()
107+
.openapi('SessionMetricsChartsDataResponse');
15108

16109
export const SessionAgentReferenceSchema = z
17110
.object({
@@ -48,11 +141,22 @@ export const SessionSchema = z
48141
created_by: z.string().describe('Caller identity that created the session (immutable).'),
49142
created_at: z.string().describe('ISO 8601 creation timestamp.'),
50143
updated_at: z.string().describe('ISO 8601 last-update timestamp.'),
144+
metrics: SessionMetricsSchema,
51145
})
52146
.openapi('Session');
53147

54148
export type SessionAgentReference = z.infer<typeof SessionAgentReferenceSchema>;
55149
export type SessionAgentInline = z.infer<typeof SessionAgentInlineSchema>;
56150
export type SessionAgent = z.infer<typeof SessionAgentSchema>;
57151
export type SessionMetrics = z.infer<typeof SessionMetricsSchema>;
152+
export type SessionMetricsPoint = z.infer<typeof SessionMetricsPointSchema>;
153+
export type SessionMetricsMeterName = z.infer<typeof SessionMetricsMeterNameSchema>;
154+
export type SessionMetricsMeter = z.infer<typeof SessionMetricsMeterSchema>;
155+
export type SessionMetricsMetersResponse = z.infer<typeof SessionMetricsMetersResponseSchema>;
156+
export type SessionMetricsChartName = z.infer<typeof SessionMetricsChartNameSchema>;
157+
export type SessionMetricsChart = z.infer<typeof SessionMetricsChartSchema>;
158+
export type SessionMetricsChartsResponse = z.infer<typeof SessionMetricsChartsResponseSchema>;
159+
export type SessionMetricsGraphLine = z.infer<typeof SessionMetricsGraphLineSchema>;
160+
export type SessionMetricsGraph = z.infer<typeof SessionMetricsGraphSchema>;
161+
export type SessionMetricsChartsDataResponse = z.infer<typeof SessionMetricsChartsDataResponseSchema>;
58162
export type Session = z.infer<typeof SessionSchema>;

‎packages/trueforge-core/src/agent-session/store/ISessionStore.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,11 @@ import type { SessionRecord } from '../models/SessionRecord';
1111
import type { TurnRecord } from '../models/TurnRecord';
1212
import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';
1313
import type { TokenPagination } from '../schemas/pagination';
14+
import type {
15+
SessionMetricsChartName,
16+
SessionMetricsChartsDataResponse,
17+
SessionMetricsMetersResponse,
18+
} from '../schemas/session';
1419
import type { CancellationReason, TerminalTurnState } from '../schemas/turn';
1520

1621
/**
@@ -61,6 +66,18 @@ export interface ListSessionsInput {
6166
created_by: string | undefined;
6267
}
6368

69+
export interface GetSessionMetricsInput {
70+
tenant_id: string;
71+
agent_id: string;
72+
created_by: string;
73+
start_timestamp: Date;
74+
end_timestamp: Date;
75+
}
76+
77+
export interface GetSessionMetricsChartDataInput extends GetSessionMetricsInput {
78+
chart_name: SessionMetricsChartName;
79+
}
80+
6481
/** Turn row fields without assembled snapshot (create input / listTurns). */
6582
export type TurnRecordWithoutSnapshot<TTurnCustom extends object = Record<string, never>> = Omit<
6683
TurnRecord<TTurnCustom>,
@@ -249,6 +266,10 @@ export interface ISessionStore<
249266
input: ListSessionsInput,
250267
): Promise<{ data: SessionRecord<TSessionCustom>[]; pagination: TokenPagination }>;
251268

269+
/** Aggregates caller-owned sessions by named agent and inclusive creation window. */
270+
getSessionMetricsMeters(input: GetSessionMetricsInput): Promise<SessionMetricsMetersResponse>;
271+
getSessionMetricsChartData(input: GetSessionMetricsChartDataInput): Promise<SessionMetricsChartsDataResponse>;
272+
252273
/**
253274
* Creates the turn AND advances `session.last_turn_id`. Context merging is the
254275
* store's responsibility: `new_context_appends` and `new_threads` are applied

‎packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import type { SessionRecord } from '../models/SessionRecord';
44
import type { TurnRecord, TurnSnapshot } from '../models/TurnRecord';
55
import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';
66
import type { TokenPagination } from '../schemas/pagination';
7+
import type { SessionMetricsChartsDataResponse, SessionMetricsMetersResponse } from '../schemas/session';
78
import type { TerminalTurnState } from '../schemas/turn';
89
import { assertCreateTurnThreadDelta } from './assertCreateTurnThreadDelta';
910
import type {
@@ -15,6 +16,8 @@ import type {
1516
DeleteSessionInput,
1617
FreezeAndGetTurnInput,
1718
GetSessionInput,
19+
GetSessionMetricsChartDataInput,
20+
GetSessionMetricsInput,
1821
GetTurnInput,
1922
ISessionStore,
2023
ListSessionEventsInput,
@@ -39,6 +42,13 @@ import {
3942
type SessionEventPageCursor,
4043
} from './SessionEventPageToken';
4144
import { decodeSessionListPageToken, paginateSessionListRows } from './SessionListPageToken';
45+
import {
46+
buildSessionMetricsChartData,
47+
buildSessionMetricsMeters,
48+
type SessionMetricsAggregate,
49+
type SessionMetricsBucket,
50+
sessionMetricsStepSeconds,
51+
} from './sessionMetrics';
4252
import {
4353
PreviousTurnRunningError,
4454
SessionAlreadyExistsError,
@@ -156,6 +166,18 @@ function paginate<T>(items: T[], limit: number, pageToken?: string): { data: T[]
156166
};
157167
}
158168

169+
function continuousPercentile(sortedValues: number[], fraction: number): number {
170+
if (sortedValues.length === 0) {
171+
return 0;
172+
}
173+
const position = fraction * (sortedValues.length - 1);
174+
const lowerIndex = Math.floor(position);
175+
const upperIndex = Math.ceil(position);
176+
const lower = sortedValues[lowerIndex] ?? 0;
177+
const upper = sortedValues[upperIndex] ?? lower;
178+
return lower + (upper - lower) * (position - lowerIndex);
179+
}
180+
159181
/**
160182
* Reference ISessionStore implementation. Proves the contract without SSE/subscription
161183
* hooks. Deep-copies on read/write boundaries so callers cannot mutate stored state.
@@ -291,6 +313,91 @@ export class InMemorySessionStore<
291313
return { data: deepCopy(page.data), pagination: page.pagination };
292314
}
293315

316+
private collectSessionMetricsData(input: GetSessionMetricsInput): {
317+
aggregate: SessionMetricsAggregate;
318+
buckets: SessionMetricsBucket[];
319+
step_seconds: number;
320+
} {
321+
const step_seconds = sessionMetricsStepSeconds(input);
322+
const aggregate: SessionMetricsAggregate = {
323+
total_sessions: 0,
324+
total_turns: 0,
325+
total_cost_in_usd: 0,
326+
active_sessions: 0,
327+
min_turns_per_session: 0,
328+
max_turns_per_session: 0,
329+
median_turns_per_session: 0,
330+
min_session_duration_ms: 0,
331+
max_session_duration_ms: 0,
332+
median_session_duration_ms: 0,
333+
p95_session_duration_ms: 0,
334+
};
335+
const buckets = new Map<number, SessionMetricsBucket>();
336+
const turnsPerSession: number[] = [];
337+
const sessionDurations: number[] = [];
338+
339+
for (const stored of this.sessions.values()) {
340+
const record = stored.record;
341+
const createdAtMs = record.created_at.getTime();
342+
if (
343+
record.tenant_id !== input.tenant_id ||
344+
record.created_by !== input.created_by ||
345+
record.agent.type !== 'reference' ||
346+
record.agent.id !== input.agent_id ||
347+
createdAtMs < input.start_timestamp.getTime() ||
348+
createdAtMs > input.end_timestamp.getTime()
349+
) {
350+
continue;
351+
}
352+
353+
aggregate.total_sessions += 1;
354+
aggregate.total_turns += record.metrics.total_turns;
355+
aggregate.total_cost_in_usd += record.metrics.total_cost_in_usd;
356+
// Turn/duration distributions only over sessions with ≥1 turn.
357+
if (record.metrics.total_turns > 0) {
358+
aggregate.active_sessions += 1;
359+
turnsPerSession.push(record.metrics.total_turns);
360+
sessionDurations.push(record.metrics.total_duration_ms);
361+
}
362+
363+
const timestamp_seconds = Math.floor(createdAtMs / 1000 / step_seconds) * step_seconds;
364+
const bucket = buckets.get(timestamp_seconds) ?? {
365+
timestamp_seconds,
366+
sessions: 0,
367+
turns: 0,
368+
cost: 0,
369+
};
370+
bucket.sessions += 1;
371+
bucket.turns += record.metrics.total_turns;
372+
bucket.cost += record.metrics.total_cost_in_usd;
373+
buckets.set(timestamp_seconds, bucket);
374+
}
375+
376+
turnsPerSession.sort((a, b) => a - b);
377+
sessionDurations.sort((a, b) => a - b);
378+
if (aggregate.active_sessions > 0) {
379+
aggregate.min_turns_per_session = turnsPerSession[0] ?? 0;
380+
aggregate.max_turns_per_session = turnsPerSession.at(-1) ?? 0;
381+
aggregate.median_turns_per_session = continuousPercentile(turnsPerSession, 0.5);
382+
aggregate.min_session_duration_ms = sessionDurations[0] ?? 0;
383+
aggregate.max_session_duration_ms = sessionDurations.at(-1) ?? 0;
384+
aggregate.median_session_duration_ms = continuousPercentile(sessionDurations, 0.5);
385+
aggregate.p95_session_duration_ms = continuousPercentile(sessionDurations, 0.95);
386+
}
387+
388+
return { aggregate, buckets: [...buckets.values()], step_seconds };
389+
}
390+
391+
async getSessionMetricsMeters(input: GetSessionMetricsInput): Promise<SessionMetricsMetersResponse> {
392+
const { aggregate } = this.collectSessionMetricsData(input);
393+
return buildSessionMetricsMeters(aggregate);
394+
}
395+
396+
async getSessionMetricsChartData(input: GetSessionMetricsChartDataInput): Promise<SessionMetricsChartsDataResponse> {
397+
const { buckets, step_seconds } = this.collectSessionMetricsData(input);
398+
return buildSessionMetricsChartData({ query: input, buckets, step_seconds });
399+
}
400+
294401
async createTurn(input: CreateTurnInput<TTurnCustom>): Promise<void> {
295402
// Atomicity is free here: this body is fully synchronous, so Node's
296403
// run-to-completion guarantees it. Real backends must still use their own

0 commit comments

Comments
 (0)