Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/llm-coding-tools-agents/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,7 @@ rstest = "0.26"
[[bench]]
name = "parser"
harness = false

[[bench]]
name = "runtime_task"
harness = false
135 changes: 135 additions & 0 deletions src/llm-coding-tools-agents/benches/runtime_task.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
//! Benchmarks for [`AgentRuntime`] task-delegation cache lookups.
//!
//! Measures the cost of [`AgentRuntime::allowed_tools`],
//! [`AgentRuntime::summarize_callable_targets`], and
//! [`AgentRuntime::can_delegate_to`] across varying agent counts.

use ahash::AHashMap;
use core::hint::black_box;
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use indexmap::IndexMap;
use llm_coding_tools_agents::{
AgentCatalog, AgentConfig, AgentMode, AgentRuntimeBuilder, AgentToolSettings, PermissionRule,
};
use llm_coding_tools_core::permissions::PermissionAction;
use llm_coding_tools_core::tool_metadata::{read as read_meta, task as task_meta};

/// Build a minimal [`AgentConfig`] for benchmark fixtures.
///
/// `permission` controls tool-access rules; all other fields are filled
/// with placeholder values suitable for performance measurement only.
fn build_agent(
name: &str,
mode: AgentMode,
permission: IndexMap<String, PermissionRule>,
) -> AgentConfig {
AgentConfig {
name: name.into(),
mode,
description: format!("{name} description").into(),
model: None,
hidden: false,
temperature: None,
top_p: None,
permission,
options: AHashMap::new(),
tool_settings: AgentToolSettings::default(),
prompt: Default::default(),
}
}

/// Create a permission map that denies all tools by default, but allows
/// pattern-matched delegation to agents named `review-*` or `worker-*`
/// via the task tool, and blanket-allows the read tool.
fn patterned_task_permission() -> IndexMap<String, PermissionRule> {
let mut patterns = IndexMap::new();
patterns.insert("*".to_string(), PermissionAction::Deny);
patterns.insert("review-*".to_string(), PermissionAction::Allow);
patterns.insert("worker-*".to_string(), PermissionAction::Allow);

IndexMap::from([
(task_meta::NAME.into(), PermissionRule::Pattern(patterns)),
(
read_meta::NAME.into(),
PermissionRule::Action(PermissionAction::Allow),
),
])
}

/// Build an [`AgentRuntime`] with one `caller` primary agent and
/// `agent_count` subordinate agents.
///
/// Subordinate names cycle through `review-NNN`, `worker-NNN`, and
/// `misc-NNN` prefixes. Every 11th subordinate is a primary-mode agent;
/// the rest are subagents.
fn build_runtime(agent_count: usize) -> llm_coding_tools_agents::AgentRuntime {
let mut agents = Vec::with_capacity(agent_count + 1);
agents.push(build_agent(
"caller",
AgentMode::Primary,
patterned_task_permission(),
));

for idx in 0..agent_count {
let name = match idx % 3 {
0 => format!("review-{idx:03}"),
1 => format!("worker-{idx:03}"),
_ => format!("misc-{idx:03}"),
};
let mode = if idx % 11 == 0 {
AgentMode::Primary
} else {
AgentMode::Subagent
};
agents.push(build_agent(&name, mode, IndexMap::new()));
}

AgentRuntimeBuilder::new()
.catalog(AgentCatalog::from_entries(agents))
.build()
}

/// Benchmark cached delegation queries against runtimes of 16, 64, and 256 agents.
///
/// Measures four operations:
/// - **allowed_tools** – full tool-set resolution for the `caller` agent.
/// - **summaries** – callable-target summary strings for `caller`.
/// - **can_delegate_hit** – pattern-match hit (`caller` → `review-003`).
/// - **can_delegate_miss** – pattern-match miss (`caller` → `misc-003`).
fn bench_runtime_task_caches(c: &mut Criterion) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
let mut group = c.benchmark_group("runtime/task_caches");

for agent_count in [16_usize, 64, 256] {
let runtime = build_runtime(agent_count);
group.throughput(Throughput::Elements(1));

group.bench_with_input(
BenchmarkId::new("allowed_tools", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.allowed_tools("caller"))),
);

group.bench_with_input(
BenchmarkId::new("summaries", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.summarize_callable_targets("caller"))),
);

group.bench_with_input(
BenchmarkId::new("can_delegate_hit", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.can_delegate_to("caller", "review-003"))),
);

group.bench_with_input(
BenchmarkId::new("can_delegate_miss", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.can_delegate_to("caller", "misc-003"))),
);
}

group.finish();
}

criterion_group!(benches, bench_runtime_task_caches);
criterion_main!(benches);
6 changes: 3 additions & 3 deletions src/llm-coding-tools-agents/src/extensions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ pub trait RulesetExt {
/// let ruleset = Ruleset::from_permission_config(&config);
/// assert!(ruleset.is_allowed("bash", "*"));
/// ```
fn from_permission_config<'a>(config: &'a IndexMap<String, PermissionRule>) -> Ruleset<'a>;
fn from_permission_config(config: &IndexMap<String, PermissionRule>) -> Ruleset;
}

impl RulesetExt for Ruleset<'_> {
fn from_permission_config<'a>(config: &'a IndexMap<String, PermissionRule>) -> Ruleset<'a> {
impl RulesetExt for Ruleset {
fn from_permission_config(config: &IndexMap<String, PermissionRule>) -> Ruleset {
let mut ruleset = Ruleset::with_capacity(config.len() * 2);

for (key, rule) in config {
Expand Down
37 changes: 36 additions & 1 deletion src/llm-coding-tools-agents/src/runtime/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,12 @@ mod tests {
use super::AgentRuntimeBuilder;
use crate::runtime::tool_catalog::{default_tools, ToolCatalogEntry, ToolCatalogKind};
use crate::runtime::AgentDefaults;
use crate::{AgentCatalog, AgentConfig, AgentMode, AgentToolSettings};
use crate::{AgentCatalog, AgentConfig, AgentMode, AgentToolSettings, PermissionRule};
use indexmap::IndexMap;
use llm_coding_tools_core::permissions::PermissionAction;
use llm_coding_tools_core::tool_metadata::{glob as glob_meta, read as read_meta};
use llm_coding_tools_core::TaskSettings;
use std::sync::Arc;

fn sample_config(name: &str, model: Option<&str>) -> AgentConfig {
AgentConfig {
Expand Down Expand Up @@ -147,4 +150,36 @@ mod tests {
assert_eq!(runtime.task_settings(), TaskSettings::default());
assert_eq!(runtime.tools(), default_tools().as_slice());
}

#[test]
fn builder_caches_permission_rulesets() {
let runtime = AgentRuntimeBuilder::new()
.catalog(AgentCatalog::from_entries([AgentConfig {
name: "planner".into(),
mode: AgentMode::Subagent,
description: "planner description".into(),
model: None,
hidden: false,
temperature: None,
top_p: None,
permission: IndexMap::from([(
read_meta::NAME.into(),
PermissionRule::Action(PermissionAction::Allow),
)]),
options: Default::default(),
tool_settings: AgentToolSettings::default(),
prompt: Default::default(),
}]))
.build();

let first = runtime
.permission_ruleset("planner")
.expect("cached ruleset should exist");
let second = runtime
.permission_ruleset("planner")
.expect("cached ruleset should exist");

assert!(Arc::ptr_eq(&first, &second));
assert!(first.is_allowed(read_meta::NAME, "*"));
}
}
88 changes: 85 additions & 3 deletions src/llm-coding-tools-agents/src/runtime/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
//! - [`AgentRuntime`] — Container for loaded agents, defaults, Task settings, and tools.
//! - [`AgentDefaults`] — Fallback settings when an agent doesn't specify them.

use super::task::resolve_allowed_tools;
use super::task::{build_runtime_task_caches, TaskTargetSummary};
use super::tool_catalog::ToolCatalogEntry;
use crate::AgentCatalog;
use crate::{AgentCatalog, RulesetExt};
use ahash::AHashMap;
use llm_coding_tools_core::permissions::Ruleset;
use llm_coding_tools_core::TaskSettings;
use std::sync::Arc;

/// Default settings used when an agent doesn't specify them.
#[derive(Debug, Clone, Default, PartialEq)]
Expand Down Expand Up @@ -40,6 +43,9 @@ pub struct AgentRuntime {
defaults: AgentDefaults,
task_settings: TaskSettings,
tools: Vec<ToolCatalogEntry>,
permission_rulesets: AHashMap<String, Arc<Ruleset>>,
allowed_tools_by_caller: AHashMap<String, Vec<ToolCatalogEntry>>,
callable_target_summaries_by_caller: AHashMap<String, Vec<TaskTargetSummary>>,
}

impl AgentRuntime {
Expand All @@ -50,11 +56,26 @@ impl AgentRuntime {
task_settings: TaskSettings,
tools: Vec<ToolCatalogEntry>,
) -> Self {
let permission_rulesets = catalog
.iter()
.map(|agent| {
(
agent.name.to_string(),
Arc::new(Ruleset::from_permission_config(&agent.permission)),
)
})
.collect();
let (allowed_tools_by_caller, callable_target_summaries_by_caller) =
build_runtime_task_caches(&catalog, &permission_rulesets, &tools);

Self {
catalog,
defaults,
task_settings,
tools,
permission_rulesets,
allowed_tools_by_caller,
callable_target_summaries_by_caller,
}
}

Expand Down Expand Up @@ -82,13 +103,74 @@ impl AgentRuntime {
&self.tools
}

/// Returns the cached permission ruleset for the named caller.
///
/// The returned [`Arc`] is cheap to clone and reuses the ruleset built when
/// the runtime was constructed.
#[inline]
pub fn permission_ruleset(&self, caller_name: &str) -> Option<Arc<Ruleset>> {
self.permission_rulesets.get(caller_name).cloned()
}

/// Returns the tool entries exposed to the named caller.
///
/// Most tools use the standard wildcard permission check (`permission -> "*"`).
/// `task` is only included when at least one `mode: all` or `mode: subagent`
/// target remains callable after applying `permission.task`.
#[inline]
pub fn allowed_tools(&self, caller_name: &str) -> Vec<ToolCatalogEntry> {
resolve_allowed_tools(self, caller_name)
self.allowed_tools_by_caller
.get(caller_name)
.cloned()
.unwrap_or_default()
}

/// Returns stable summaries for every agent the named caller may delegate to via Task.
///
/// Only agents with [`AgentMode::All`] or [`AgentMode::Subagent`] are
/// considered. When the caller defines `permission.task`, targets are
/// filtered by those rules; otherwise all non-primary agents are included.
/// Results are sorted alphabetically by target name.
///
/// # Arguments
///
/// * `caller_name` - Name of the agent that wants to delegate.
///
/// # Returns
///
/// A [`TaskTargetSummary`] per callable target. Empty if `caller_name`
/// is not in the catalog or no targets survive permission filtering.
///
/// [`AgentMode::All`]: crate::AgentMode::All
/// [`AgentMode::Subagent`]: crate::AgentMode::Subagent
#[inline]
pub fn summarize_callable_targets(&self, caller_name: &str) -> Vec<TaskTargetSummary> {
self.callable_target_summaries_by_caller
.get(caller_name)
.cloned()
.unwrap_or_default()
}

/// Returns whether the named caller may delegate to the given target.
///
/// Looks up the caller's filtered callable-target list and performs a
/// binary search for `target_name`.
///
/// # Arguments
/// - `caller_name`: Name of the agent that would originate the delegation.
/// - `target_name`: Name of the candidate delegate target.
///
/// # Returns
/// - `true` if `target_name` appears in the caller's permitted target list.
/// - `false` if the caller is not in the catalog or the target is absent.
#[inline]
pub fn can_delegate_to(&self, caller_name: &str, target_name: &str) -> bool {
self.callable_target_summaries_by_caller
.get(caller_name)
.is_some_and(|summaries| {
summaries
.binary_search_by(|summary| summary.name.as_ref().cmp(target_name))
.is_ok()
})
}
}
Loading