Skip to content

Commit d566521

Browse files
authored
feat(explore): record quantitative results and return scoped feedback (#7011)
Record scoped quantitative observations on existing Explore findings and return best/regression feedback through history and next-turn context. Signed-off-by: LoopX Agent <337587101+loopx-agent@users.noreply.github.com>
1 parent 5bf58c1 commit d566521

17 files changed

Lines changed: 676 additions & 16 deletions

File tree

‎apps/presentation/dashboard/src/data/chat.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { HANDOFF_MODES, EXECUTION_HANDOFF_MODES, type ExecutionHandoffMode } fro
33
import { normalizeGoalDraft } from "../../../../../loopx/control_plane/collaboration/goal_draft.js";
44
import { parseTurnStep, type TurnStep } from "./turn-steps.js";
55
import { z } from "zod";
6+
import {normalizeExploreMeasurements} from "../../../../../loopx/control_plane/capabilities/explore_measurements.js";
67
import { actionSourceBasisSchema, todoUpdateSourceBasisSchema } from "./action-source-basis.js";
78

89
import {
@@ -2665,6 +2666,7 @@ const exploreResultPageSchema = z.object({
26652666
finding_id: z.string(), finding: z.string(), summary: z.string(), status: z.string(),
26662667
node_id: z.string(), question: z.string(), scope: z.string(), agent_id: z.string(),
26672668
evidence_refs: z.array(z.string()), last_updated_at: z.string(),
2669+
measurements: z.unknown().transform(normalizeExploreMeasurements).optional(),
26682670
linked_todos: z.array(z.object({
26692671
todo_id: z.string(), text: z.string(), status: z.string(), claimed_by: z.string(),
26702672
})),
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import type {ExploreMeasurement, ExploreMeasurementFeedback} from "../../../../../../loopx/control_plane/capabilities/explore_measurements.js";
2+
3+
export const formatExploreMeasurementValue = (value: number | null, zh: boolean) => value === null
4+
? (zh ? "未测量" : "Not measured")
5+
: new Intl.NumberFormat(zh ? "zh-CN" : "en", {maximumSignificantDigits: 12}).format(value);
6+
7+
/** Values and comparisons come from Explore's shared owner, never a UI scorer. */
8+
export function ExploreMeasurements({measurements, feedback = [], zh, title}: {
9+
measurements: ExploreMeasurement[]; feedback?: ExploreMeasurementFeedback[]; zh: boolean; title?: string;
10+
}) {
11+
if (!measurements.length) return null;
12+
const number = (value: number | null) => formatExploreMeasurementValue(value, zh);
13+
const label = title ?? (zh ? "量化结果" : "Measured results");
14+
return <section className="explore-measurements" aria-label={label}>
15+
<h4>{label}</h4>
16+
{measurements.map((measurement, index) => {
17+
const comparison = feedback.find(item => item.metric === measurement.metric && item.unit === measurement.unit
18+
&& item.higher_is_better === measurement.higher_is_better && item.comparison_scope === measurement.comparison_scope
19+
&& item.input_revision === measurement.input_revision && item.evaluation_ref === measurement.evaluation_ref);
20+
const delta = comparison?.improvement;
21+
return <div key={index}>
22+
<p><strong>{measurement.metric} · {number(measurement.value)} {measurement.unit}</strong>
23+
<small> · {measurement.higher_is_better ? (zh ? "越高越好" : "Higher is better") : (zh ? "越低越好" : "Lower is better")}</small></p>
24+
{comparison ? <p>{zh ? "同口径 best" : "Best in this scope"}: {number(comparison.best?.value ?? null)}
25+
{delta !== null && delta !== undefined ? <> · {zh ? "相对先前 best 的改善" : "Improvement over prior best"}: {delta > 0 ? "+" : ""}{number(delta)}</> : null}</p> : null}
26+
<details><summary>{zh ? "评测范围与来源" : "Evaluation scope and sources"}</summary>
27+
<p>{zh ? "比较范围" : "Comparison scope"}: {measurement.comparison_scope}<br/>
28+
{zh ? "输入" : "Input"}: {measurement.input_revision}<br/>
29+
{zh ? "评测" : "Evaluation"}: {measurement.evaluation_ref}<br/>
30+
{zh ? "候选" : "Candidate"}: {measurement.candidate_ref}<br/>
31+
{zh ? "结果引用" : "Result reference"}: {measurement.evidence_ref}</p>
32+
{comparison?.best ? <p>{zh ? "best 候选 / 来源" : "Best candidate / source"}: {comparison.best.candidate_ref} · {comparison.best.evidence_ref}</p> : null}
33+
<p>{zh ? "同一输入、评测和预算条件下的来源记录;best 不代表当前保留版本或验收通过。" : "Source-recorded values under the same input, evaluation and budget conditions; best does not establish the retained version or acceptance."}</p>
34+
</details>
35+
</div>;
36+
})}
37+
</section>;
38+
}

‎apps/presentation/dashboard/src/features/personal-workspace/goal-explore-results.tsx‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {useEffect, useRef, useState} from "react";
22
import {RefreshCw} from "lucide-react";
33
import {fetchExploreResults, type ExploreResultPage} from "../../data/chat";
4+
import {ExploreMeasurements} from "./explore-measurements";
45

56
/** Read-only saved evidence, including history when Explore has been disabled. */
67
export function GoalExploreResults({goalId, zh}: {goalId: string; zh: boolean}) {
@@ -44,6 +45,7 @@ export function GoalExploreResults({goalId, zh}: {goalId: string; zh: boolean})
4445
{selected ? <div ref={reader} tabIndex={-1} className="goal-team-result-reader" style={{overflowWrap: "anywhere"}}>
4546
<h4>{selected.question || selected.finding}</h4>
4647
{selected.scope ? <p>{zh ? "问题范围" : "Question scope"}: {selected.scope}</p> : null}
48+
{selected.measurements ? <ExploreMeasurements measurements={selected.measurements} zh={zh}/> : null}
4749
<p style={{whiteSpace: "pre-wrap", overflowWrap: "anywhere"}}>{selected.summary}</p>
4850
<p>{zh ? "记录状态" : "Recorded status"}: {selected.status}</p>
4951
<p>{zh ? "记录者" : "Recorded by"}: {selected.agent_id || "—"}</p>

‎apps/presentation/dashboard/src/features/personal-workspace/goal-research-history.tsx‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {sourceReadApiUrl} from "../../data/local-status-query";
99
import {ContextHelp} from "./context-help";
1010
import {groupUnlinkedHistoryQuestions, layoutExploreHistory, parseExploreHistoryPage, projectExploreHistory, type ExploreHistoryEvent} from "../../../../../../loopx/control_plane/presentation/explore_history.js";
1111
import "./goal-research-history.css";
12+
import {ExploreMeasurements, formatExploreMeasurementValue} from "./explore-measurements";
1213

1314
type Selection = {nodes: string[]; day: string | null};
1415
type Lens = "timeline" | "graph";
@@ -78,7 +79,7 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
7879
const decision = sections?.decision;
7980
const execution = finding.execution_observation;
8081
const measured = (value: unknown) => value === null ? (zh ? "未测量" : "Not measured") : String(value);
81-
return <>{sections ? <dl className="research-finding-sections">
82+
return <>{finding.measurements ? <ExploreMeasurements measurements={finding.measurements} zh={zh}/> : null}{sections ? <dl className="research-finding-sections">
8283
<dt>{sectionCopy.observation}</dt><dd>{sections.observation}</dd>
8384
{decision ? <>
8485
<dt>{zh ? "决策" : "Decision"}</dt><dd><b className={`research-outcome is-${decision.outcome}`}>{outcomeCopy[decision.outcome] ?? decision.outcome}</b></dd>
@@ -99,7 +100,7 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
99100
</details> : null}</>;
100101
};
101102
const findingElement = (finding: ExploreHistoryEvent) => <div className="research-recorded-finding" key={finding.event_id}><strong>{finding.title}</strong><small>{timeLabel(finding.recorded_at)} · {zh ? "来源标记" : "Source label"} · {stateLabel(finding.status)}{finding.confidence !== null ? ` · ${zh ? "来源自报置信度" : "Source-reported confidence"} ${finding.confidence}` : ""}</small>{findingBody(finding)}{finding.evidence_refs.map(ref => <code key={ref}>{ref}</code>)}</div>;
102-
const updateRecords = (events: readonly ExploreHistoryEvent[]) => <ol className="research-history-records">{events.map(event => <li key={event.event_id}><time>{timeLabel(event.recorded_at)}</time><span>{stateLabel(event.status)} · {event.title}{event.summary ? ` — ${event.summary}` : ""}</span>{event.parent_id ? <code>{copy.group}: {event.parent_id}</code> : null}{event.evidence_refs.map(ref => <code key={ref}>{ref}</code>)}{event.run_id ? <code>{event.run_id}</code> : null}</li>)}</ol>;
103+
const updateRecords = (events: readonly ExploreHistoryEvent[]) => <ol className="research-history-records">{events.map(event => <li key={event.event_id}><time>{timeLabel(event.recorded_at)}</time><span>{stateLabel(event.status)} · {event.title}{event.summary ? ` — ${event.summary}` : ""}</span>{event.measurements ? <ExploreMeasurements measurements={event.measurements} zh={zh}/> : null}{event.parent_id ? <code>{copy.group}: {event.parent_id}</code> : null}{event.evidence_refs.map(ref => <code key={ref}>{ref}</code>)}{event.run_id ? <code>{event.run_id}</code> : null}</li>)}</ol>;
103104

104105
useEffect(() => {
105106
const pop = () => setSelection(selectedFromUrl(goalId));
@@ -147,6 +148,12 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
147148
catch (failure) {return {value: null, error: String(failure)};}
148149
}, [read.events, goalId]);
149150
const value = history.value;
151+
const measurementFeedback = value?.measurementFeedback ?? [];
152+
const measurementSummary = (nodeId: string) => {
153+
const feedback = measurementFeedback.filter(item => item.node_id === nodeId);
154+
return feedback.length ? <ExploreMeasurements title={zh ? "最近记录与历史最佳" : "Latest recorded and historical best"}
155+
measurements={feedback.map(item => item.latest.measurement)} feedback={feedback} zh={zh}/> : null;
156+
};
150157
const grouped = useMemo(() => groupUnlinkedHistoryQuestions(value?.nodes ?? [], value?.edges ?? []), [value]);
151158
const layout = useMemo(() => layoutExploreHistory(grouped.nodes, value?.edges ?? []), [grouped, value]);
152159
const timeline = useMemo(() => value ? buildExploreTimeline(value.nodes, value.edges, value.events) : null, [value]);
@@ -223,11 +230,15 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
223230
return <path key={`parent:${item.node.id}`} className="is-grouping" d={`M${parent.x + 206},${parent.y + 47} C${parent.x + 226},${parent.y + 47} ${item.x - 20},${item.y + 47} ${item.x},${item.y + 47}`} />;
224231
})}
225232
</svg>
226-
{layout.map(({node, x, y}) => <button key={node.id} type="button" className={`research-history-node is-${node.latest.status}${selection.nodes.includes(node.id) ? " is-selected" : ""}`}
233+
{layout.map(({node, x, y}) => {
234+
const measured = measurementFeedback.find(item => item.node_id === node.id);
235+
return <button key={node.id} type="button" className={`research-history-node is-${node.latest.status}${selection.nodes.includes(node.id) ? " is-selected" : ""}`}
227236
style={{left: x, top: y}} aria-pressed={selection.nodes.includes(node.id)} aria-label={`${node.latest.title} · ${stateLabel(node.latest.status)}`} onClick={() => choose(node.id)}>
228237
<span className="research-node-meta">{node.latest.node_kind === "experiment" ? <FlaskConical size={13} aria-hidden/> : <Circle size={12} aria-hidden/>}{kindLabel(node.latest.node_kind)}<small>{stateLabel(node.latest.status)}</small></span>
229-
<strong>{node.latest.title}</strong><span className="research-node-count"><time>{timeLabel(node.updates[0]?.recorded_at)}</time>{node.updates.length > 1 ? ` · ${node.updates.length} ${copy.changes}` : ""}{node.findings.length ? ` · ${node.findings.length} ${copy.findings}` : ""}</span>
230-
</button>)}
238+
<strong>{node.latest.title}</strong>{measured ? <span className="research-node-count" title={`${measured.metric} (${measured.unit}) · ${measured.comparison_scope} · ${measured.evaluation_ref}`}>
239+
{measured.metric} {formatExploreMeasurementValue(measured.latest.measurement.value, zh)} · best {formatExploreMeasurementValue(measured.best?.value ?? null, zh)} {measured.higher_is_better ? "↑" : "↓"}
240+
</span> : <span className="research-node-count"><time>{timeLabel(node.updates[0]?.recorded_at)}</time>{node.updates.length > 1 ? ` · ${node.updates.length} ${copy.changes}` : ""}{node.findings.length ? ` · ${node.findings.length} ${copy.findings}` : ""}</span>}
241+
</button>})}
231242
{grouped.groups.map((group, index) => <button type="button" key={`day:${group.day}`} className={`research-history-node research-history-group${selectedGroup === group ? " is-selected" : ""}`} style={{left: (index % 4) * 230 + 24, top: clusterY + Math.floor(index / 4) * 124}}
232243
aria-pressed={selectedGroup === group} onClick={() => select({nodes: [], day: group.day})}>
233244
<span className="research-node-meta"><Circle size={12} aria-hidden/>{zh ? "未关联问题" : "Unlinked questions"}</span>
@@ -239,6 +250,7 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
239250
<p className="research-detail-kind"><GitBranch size={14} aria-hidden/>{kindLabel(selected.latest.node_kind)} · {stateLabel(selected.latest.status)}</p>
240251
<h3>{selected.latest.title}</h3><p>{selected.latest.summary || (zh ? "此节点尚无详细说明。" : "No detail was recorded for this node.")}</p>
241252
{selected.latest.blocked_reason ? <p className="research-history-error">{selected.latest.blocked_reason}</p> : null}
253+
{measurementSummary(selected.id)}
242254
<h4>{copy.evidence}</h4>
243255
{selected.findings.length ? selected.findings.map(findingElement) : <p>{zh ? "此节点没有附属结论记录。" : "No finding is attached to this node."}</p>}
244256
<h4>{copy.versions}</h4>{selected.latest.evidence_refs.length ? selected.latest.evidence_refs.map(ref => <code key={ref}>{ref}</code>) : <p>{zh ? "产物版本与来源引用未记录。" : "Artifact version and source references were not recorded."}</p>}
@@ -249,6 +261,7 @@ export function GoalResearchHistory({goalId, active, zh, refreshRevision = 0, re
249261
{selectedMembers?.length ? <div className="research-history-detail" ref={detail} role="region" aria-label={selectedGroup ? (zh ? "按日期收纳的未关联问题" : "Unlinked questions by date") : (zh ? "相邻研究项的全部记录" : "All records for nearby research items")} tabIndex={-1} onKeyDown={event => {if (event.key === "Escape") {event.stopPropagation(); choose("", false); graph.current?.focus();}}}>
250262
<h3>{selectedGroup ? `${selectedGroup.day || copy.unknown} · ${zh ? "未关联问题" : "Unlinked questions"}` : (zh ? `相邻研究项 · ${selectedMembers.length}` : `Nearby research items · ${selectedMembers.length}`)}</h3>
251263
{selectedMembers.map(node => <article className="research-history-group-member" key={node.id} data-research-node={node.id}><h4>{node.latest.title}</h4><p>{stateLabel(node.latest.status)} · {node.latest.summary}</p>{node.latest.evidence_refs.map(ref => <code key={ref}>{ref}</code>)}
264+
{measurementSummary(node.id)}
252265
{node.findings.map(findingElement)}
253266
{updateRecords(node.revisions)}
254267
</article>)}

‎loopx/capabilities/explore/README.md‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,46 @@ This records producer evidence, not verified execution or utility. Findings,
1212
replan context and the existing research-history detail retain it; omission
1313
preserves old events. See the [local probe producer](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md).
1414

15+
## Quantitative findings
16+
17+
An ordinary result attachment can include `measurements`, saved on its existing
18+
node-linked finding. The graph and research-history detail retain the values;
19+
`explore turn-context` supplies bounded `measurement_feedback` to the next turn.
20+
Manual capture uses `explore finding --node <id> --evidence-ref result:a
21+
--measurements-json measurements.json`. The file contains one to eight items:
22+
23+
```json
24+
[{"metric":"quality","value":8,"unit":"points","higher_is_better":true,
25+
"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1",
26+
"evaluation_ref":"evaluator:v1","candidate_ref":"candidate:a","evidence_ref":"result:a"}]
27+
```
28+
29+
Each result reference must belong to the finding's `evidence_refs`; ordinary
30+
attachments also require the same `input_revision`. Use a new finding id per
31+
evaluated candidate, and reuse an id only to correct that finding. Missing values
32+
are explicit `null`; zero and negative values remain valid measurements.
33+
34+
Comparisons require the same node, metric, unit, direction, comparison scope,
35+
input revision and evaluator. Producers must change the scope when task,
36+
model/resource budget or evaluation conditions change, and separate native
37+
evaluation from local proxies. Explore validates the declared comparison keys,
38+
not their truth. It computes the latest recorded measurement, historical best
39+
candidate/source and direction-adjusted improvement against the prior best,
40+
before applying the context's display limit. Latest follows source-log append
41+
order, including tied or backfilled timestamps; its ordinal is derived, not
42+
another persisted fact. Corrected findings replace earlier
43+
versions in comparisons; original versions remain in history. Counts are records,
44+
not independent trials. Feedback pages reuse `--result-limit` (1..20, default 3),
45+
`--result-offset`, `--result-revision` and the optional `--result-node` filter.
46+
Follow `measurement_feedback_page.next_command` for additional groups, including
47+
multiple metrics or conditions on one node. The shared revision binds all compared
48+
sources, including manual findings; a changed source requires restarting the read.
49+
The next-turn guidance uses this feedback to investigate
50+
regressions or revisit better candidates before repeating work. Best does not
51+
identify the current retained candidate, establish causality, satisfy acceptance
52+
or authorize execution. Actual adoption and outcome benefit require subsequent
53+
work and evidence; automatic branch rankings remain unchanged.
54+
1555
## At a Glance
1656

1757
Explore Harness is a supported, default-off optional capability for

‎loopx/capabilities/explore/README.zh-CN.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,38 @@
99
不认证执行真实性或收益;finding、replan 和既有研究历程详情保留相同信息,缺省仍兼容
1010
旧事件。参见[本地探针生产路径](../../../benchmark/runtime/EXECUTION_OBSERVATIONS.md)。
1111

12+
## 量化结论
13+
14+
普通结果附件可附带 `measurements`,保存到既有节点所属的 finding。关系图和
15+
研究历程详情保留分数;`explore turn-context` 将有界的 `measurement_feedback`
16+
交给下一轮。手动记录使用 `explore finding --node <id> --evidence-ref result:a
17+
--measurements-json measurements.json`,文件包含一至八项:
18+
19+
```json
20+
[{"metric":"quality","value":8,"unit":"points","higher_is_better":true,
21+
"comparison_scope":"validation:fixed-budget-v1","input_revision":"dataset-v1",
22+
"evaluation_ref":"evaluator:v1","candidate_ref":"candidate:a","evidence_ref":"result:a"}]
23+
```
24+
25+
结果引用必须属于 finding 的 `evidence_refs`;普通附件还要求相同 `input_revision`。
26+
每个受评测候选使用新 finding id,仅纠正同一结论时复用 id。未测量显式记为 `null`,
27+
零分和负分仍为有效测量。
28+
29+
只有节点、指标、单位、方向、比较范围、输入版本和评测器全部一致时才比较。
30+
任务、模型或资源预算、评测条件改变时,生产者必须更换比较范围;原生评测与本地
31+
代理指标分别记录。Explore 校验声明的比较键,不认证声明真实性。它在上下文裁剪前
32+
计算最近记录的测量、历史最佳候选及来源、相对先前最佳的方向归一化改善幅度。
33+
最近记录按源日志追加顺序确定;时间相同或事后回填不改变此顺序,序号仅从日志
34+
派生,不另行持久化。
35+
纠正结论会在比较中替换旧版本,原始版本仍可在历史中读取;计数表示记录数,
36+
不表示独立试验次数。反馈分页复用 `--result-limit`(1..20,默认 3)、
37+
`--result-offset`、`--result-revision` 和可选的 `--result-node` 过滤。
38+
沿 `measurement_feedback_page.next_command` 读取其余组,同一节点的多个指标或条件
39+
也可完整翻页。共同 revision 绑定全部比较来源,包含手动 finding;来源变化须从头读取。
40+
下一轮指导使用这些反馈排查退步、回看更好候选,再决定是否
41+
重复尝试。best 不代表当前保留版本、因果收益、验收通过或执行授权;是否实际采用
42+
并产生收益,仍需后续工作和证据确认。自动分支排序保持原有行为。
43+
1244
## 概览
1345

1446
LoopX Explore 是面向长程探索目标(软件研究、安全攻击面测绘、领域调研)的受支持、默认关闭的可选能力。它把“到处看看”变成有界、可观测、有门控的流程,包含三根支柱:

0 commit comments

Comments
 (0)