Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion src/app/components/chat-panel/chat-panel.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,22 @@ <h3 style="margin: 0; color: var(--mat-sys-primary);">Evaluation Result</h3>
</div>
</div>
}
@if (uiEvents.length === 0 && agentReadme) {
@if (uiEvents.length === 0 && agentReadme && !isEvalResult) {
<div class="readme-content">
<ng-container [ngComponentOutlet]="markdownComponent"
[ngComponentOutletInputs]="{text: agentReadme, thought: false, isReadme: true}"></ng-container>
</div>
}
@if (uiEvents.length === 0 && isEvalResult) {
<div class="empty-eval-result">
<mat-icon>info_outline</mat-icon>
<p>This eval run produced no conversation turns.</p>
<p class="empty-eval-result-hint">
The run likely failed before any turns were recorded. Check the run status
and the server logs, then try running it again.
</p>
</div>
}
@for (item of displayItems; track item; let i = $index) {
@if (item.type === 'event') {
@let uiEvent = item.event;
Expand Down
28 changes: 28 additions & 0 deletions src/app/components/chat-panel/chat-panel.component.scss
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,34 @@ button.send-message-btn {
}
}

.empty-eval-result {
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
gap: 8px;
padding: 40px 20px;
text-align: center;
color: var(--mat-sys-on-surface-variant);

mat-icon {
font-size: 32px;
width: 32px;
height: 32px;
}

p {
margin: 0;
font-size: 14px;
}

.empty-eval-result-hint {
max-width: 360px;
font-size: 12px;
opacity: 0.85;
}
}

.branches-container {
margin: 8px -20px;
border-radius: 8px;
Expand Down
2 changes: 2 additions & 0 deletions src/app/components/chat-panel/chat-panel.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,8 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit {
@Input() userId: string = '';
@Input() sessionId: string = '';
@Input() viewMode: 'events' | 'traces' = 'events';
// Renders the empty-run message instead of the README when there are no events.
@Input() isEvalResult: boolean = false;
@Input() shouldShowEvent?: (uiEvent: UiEvent) => boolean;
spansByInvocationId = new Map<string, Span[]>();
displayItems: DisplayItem[] = [];
Expand Down
26 changes: 25 additions & 1 deletion src/app/components/chat/chat.component.html
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,7 @@
</div>
} } @if (appName != "") {
<div class="chat-sub-toolbar">
@if (!(uiEvents().length === 0 && agentReadme)) {
@if (!(uiEvents().length === 0 && agentReadme && chatType() !== 'eval-result')) {
<mat-button-toggle-group [value]="viewMode()" (change)="onViewModeChange($event.value)" hideSingleSelectionIndicator>
<mat-button-toggle value="events">Events</mat-button-toggle>
<mat-button-toggle value="traces">Traces</mat-button-toggle>
Expand Down Expand Up @@ -625,6 +625,7 @@
<app-chat-panel
[appName]="appName"
[agentReadme]="agentReadme"
[isEvalResult]="chatType() === 'eval-result'"
[(userInput)]="userInput"
[hideIntermediateEvents]="hideIntermediateEvents()"
[uiEvents]="filteredUiEvents()"
Expand Down Expand Up @@ -670,6 +671,7 @@
<app-chat-panel
[appName]="appName"
[agentReadme]="agentReadme"
[isEvalResult]="chatType() === 'eval-result'"
[hideIntermediateEvents]="hideIntermediateEvents()"
[uiEvents]="filteredUiEvents()"
[showBranches]="showBranches()"
Expand Down Expand Up @@ -742,6 +744,25 @@
style="margin-top: 8px; border-top: 1px solid var(--mat-sys-outline-variant); padding-top: 6px; margin-bottom: 0;">
{{ desc }}</div>
}
@if (metric.details?.explanation) {
<div class="tooltip-explanation"
style="margin-top: 8px; border-top: 1px solid var(--mat-sys-outline-variant); padding-top: 6px;">
{{ metric.details.explanation }}</div>
}
@if (metric.details?.rubricScores?.length) {
<div class="tooltip-rubrics"
style="margin-top: 8px; border-top: 1px solid var(--mat-sys-outline-variant); padding-top: 6px;">
<div class="tooltip-label" style="margin-bottom: 4px;">Rubrics</div>
@for (rubric of metric.details.rubricScores; track $index) {
<div class="tooltip-rubric" style="display: flex; gap: 6px; align-items: flex-start; margin-bottom: 4px;">
<span class="material-symbols-outlined"
[style.color]="rubricPassed(rubric) ? '#2e7d32' : 'var(--mat-sys-error)'"
style="font-size: 14px; line-height: 16px;">{{ rubricPassed(rubric) ? 'check_circle' : 'cancel' }}</span>
<span style="font-size: 11px;">{{ rubric.rationale || rubric.rubricId }}</span>
</div>
}
</div>
}
</div>
</div>
}
Expand All @@ -756,6 +777,7 @@
<app-chat-panel
[appName]="appName"
[agentReadme]="agentReadme"
[isEvalResult]="chatType() === 'eval-result'"
[hideIntermediateEvents]="hideIntermediateEvents()"
[uiEvents]="filteredExpectedUiEvents()"
[showBranches]="showBranches()"
Expand Down Expand Up @@ -786,6 +808,7 @@
<app-chat-panel
[appName]="appName"
[agentReadme]="agentReadme"
[isEvalResult]="chatType() === 'eval-result'"
[hideIntermediateEvents]="hideIntermediateEvents()"
[uiEvents]="filteredUiEvents()"
[showBranches]="showBranches()"
Expand Down Expand Up @@ -839,6 +862,7 @@
<app-chat-panel
[appName]="appName"
[agentReadme]="agentReadme"
[isEvalResult]="chatType() === 'eval-result'"
[hideIntermediateEvents]="hideIntermediateEvents()"
[uiEvents]="filteredUiEvents()"
[showBranches]="showBranches()"
Expand Down
1 change: 0 additions & 1 deletion src/app/components/chat/chat.component.scss
Original file line number Diff line number Diff line change
Expand Up @@ -1398,4 +1398,3 @@ app-canvas {
.spinning {
animation: spin 1s linear infinite;
}

214 changes: 212 additions & 2 deletions src/app/components/chat/chat.component.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,14 @@ describe('ChatComponent', () => {
expect(component).toBeTruthy();
});

it('rubricPassed reads backend verdict, falling back to score', () => {
expect(component.rubricPassed({verdict: true})).toBe(true);
expect(component.rubricPassed({verdict: false, score: 1})).toBe(false);
expect(component.rubricPassed({score: 1})).toBe(true);
expect(component.rubricPassed({score: 0})).toBe(false);
expect(component.rubricPassed({})).toBe(false);
});

it(
'should pre-fill user input from "q" query param only when app is selected',
fakeAsync(() => {
Expand Down Expand Up @@ -728,6 +736,208 @@ describe('ChatComponent', () => {
});
});

describe('live eval result rendering', () => {
// A live run's persisted session is a noisy stream of audio chunks,
// partial transcriptions, and turn-complete markers.
const noisyLiveSessionEvents = [
{
id: 'e1',
author: 'user',
content: {
parts: [
{text: 'hello'},
{inlineData: {mimeType: 'audio/pcm', data: 'AAAA'}},
],
},
},
{
id: 'e2',
author: 'bot',
content: {parts: [{inlineData: {mimeType: 'audio/pcm', data: 'BBBB'}}]},
},
{id: 'e3', author: 'bot', outputTranscription: {text: 'hi there'}},
{id: 'e4', author: 'bot', content: {parts: []}},
];

// A valid base64 PCM chunk (4 bytes -> 2 16-bit samples).
const pcmChunk = btoa('\x01\x02\x03\x04');

// The clean per-invocation view the backend also returns. The first turn
// streams its model audio across many chunk events (mirroring a real live
// run) plus one real tool call.
const evalCaseResult = {
evalMetricResultPerInvocation: [
{
actualInvocation: {
userContent: {
parts: [
{text: 'hello'},
{inlineData: {mimeType: 'audio/pcm', data: pcmChunk}},
],
},
finalResponse: {parts: [{text: 'hi there'}]},
intermediateData: {
invocationEvents: [
{
id: 'ie-fc',
author: 'bot',
content: {
parts: [{functionCall: {name: 'do_thing', args: {}}}],
},
},
{
id: 'ie-a1',
author: 'bot',
content: {parts: [{inlineData: {mimeType: 'audio/pcm', data: pcmChunk}}]},
},
{
id: 'ie-a2',
author: 'bot',
content: {parts: [{inlineData: {mimeType: 'audio/pcm', data: pcmChunk}}]},
},
{
id: 'ie-partial',
author: 'bot',
partial: true,
content: {parts: [{inlineData: {mimeType: 'audio/pcm', data: pcmChunk}}]},
},
],
},
},
evalMetricResults: [
{metricName: 'response_match_score', evalStatus: 1, score: 1, threshold: 0.7},
],
},
{
actualInvocation: {
userContent: {parts: [{text: 'and again'}]},
finalResponse: {parts: [{text: 'sure thing'}]},
intermediateData: {},
},
evalMetricResults: [
{metricName: 'response_match_score', evalStatus: 2, score: 0.2, threshold: 0.7},
],
},
],
};

function liveResultSession() {
return {
id: SESSION_1_ID,
state: {},
events: noisyLiveSessionEvents,
isEvalResult: true,
evalCaseResult: {
...evalCaseResult,
sessionDetails: {events: noisyLiveSessionEvents},
},
};
}

beforeEach(() => {
mockEventService.getTraceResponse.next([]);
});

it('detects a live eval result from audio/transcription session events',
() => {
expect(component['isLiveEvalResult'](liveResultSession().evalCaseResult))
.toBeTrue();
expect(component['isLiveEvalResult']({sessionDetails: {events: []}}))
.toBeFalse();
});

it('renders one bubble per turn (+tool call), not per raw audio chunk', () => {
component['updateWithSelectedSession'](liveResultSession() as any);
fixture.detectChanges();

// Turn 1: user + tool-call bot event + response bot event.
// Turn 2: user + response bot event. The many raw audio-chunk events
// (and the partial) are folded/dropped, NOT rendered as bubbles.
const roles = component.uiEvents().map((e: any) => e.role);
expect(roles).toEqual(['user', 'bot', 'bot', 'user', 'bot']);
});

it('renders transcript text and the tool call', () => {
component['updateWithSelectedSession'](liveResultSession() as any);
fixture.detectChanges();

const events = component.uiEvents();
expect(events[0]).toEqual(jasmine.objectContaining({role: 'user', text: 'hello'}));
// The tool call survives; audio-only chunk events do not.
expect(events[1].functionCalls?.[0]?.name).toBe('do_thing');
expect(events[2]).toEqual(jasmine.objectContaining({role: 'bot', text: 'hi there'}));
});

it('folds the turn\'s PCM chunks into one playable WAV clip', () => {
component['updateWithSelectedSession'](liveResultSession() as any);
fixture.detectChanges();

const events = component.uiEvents();
// Exactly one audio clip in the whole conversation (turn 1's response),
// not one per raw chunk.
const audioEvents = events.filter(
(e: any) => e.inlineData?.mimeType?.startsWith('audio/'));
expect(audioEvents.length).toBe(1);
// It is a playable WAV data URL (not raw audio/pcm).
expect(audioEvents[0].inlineData!.mimeType).toBe('audio/wav');
expect(audioEvents[0].inlineData!.data).toContain('data:audio/wav;base64,');
// No unplayable raw-PCM parts leak through.
expect(events.some((e: any) => e.inlineData?.mimeType === 'audio/pcm'))
.toBeFalse();
// The clip rides on turn 1's response bubble (which also has text).
expect(audioEvents[0].text).toBe('hi there');
});

it('attaches per-turn PASS/FAIL to live bot bubbles, as non-live does',
() => {
component['updateWithSelectedSession'](liveResultSession() as any);
fixture.detectChanges();

// Bubbles: [user, bot tool call, bot response] for the passing turn
// 1, then [user, bot response] for the failing turn 2.
const events = component.uiEvents();
expect(events[1].evalStatus).toBe(1);
expect(events[2].evalStatus).toBe(1);
expect(events[4].evalStatus).toBe(2);

// User bubbles are never annotated.
expect(events[0].evalStatus).toBeUndefined();
expect(events[3].evalStatus).toBeUndefined();
});

it('attaches the failing metric detail to the live response bubble', () => {
component['updateWithSelectedSession'](liveResultSession() as any);
fixture.detectChanges();

// Turn 2's response failed response_match_score; its bubble carries the
// "Match score / Threshold" detail, as in a non-live result.
const failedResponse = component.uiEvents()[4];
expect(failedResponse.failedMetric).toBe('response_match_score');
expect(failedResponse.evalScore).toBe(0.2);
expect(failedResponse.evalThreshold).toBe(0.7);
});

it('falls back to raw session events for a non-live eval result', () => {
const nonLive = {
id: SESSION_1_ID,
state: {},
events: [
{id: 'u', author: 'user', content: {parts: [{text: 'q'}]}},
{id: 'b', author: 'bot', content: {parts: [{text: 'a'}]}},
],
isEvalResult: true,
evalCaseResult: {
evalMetricResultPerInvocation: [],
sessionDetails: {events: []},
},
};
component['updateWithSelectedSession'](nonLive as any);
fixture.detectChanges();

expect(component.uiEvents().map((e: any) => e.text)).toEqual(['q', 'a']);
});
});

describe('UI and State', () => {
beforeEach(() => {
mockAgentService.listAppsResponse.next(
Expand Down Expand Up @@ -1213,7 +1423,7 @@ describe('ChatComponent', () => {
it('should save message', () => {
expect(mockMessage.text).toBe(NEW_RESPONSE);
expect(component.hasEvalCaseChanged()).toBe(true);
expect(component.updatedEvalCase!.conversation[0]
expect(component.updatedEvalCase!.conversation![0]
.finalResponse!.parts![0]!.text)
.toBe(NEW_RESPONSE);
});
Expand All @@ -1230,7 +1440,7 @@ describe('ChatComponent', () => {
it('should delete message', () => {
expect(component.uiEvents().length).toBe(0);
expect(component.hasEvalCaseChanged()).toBe(true);
expect(component.updatedEvalCase!.conversation[0]
expect(component.updatedEvalCase!.conversation![0]
.finalResponse!.parts!.length)
.toBe(0);
});
Expand Down
Loading
Loading