Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/llm-coding-tools-agents/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,7 @@ rstest = "0.26"
[[bench]]
name = "parser"
harness = false

[[bench]]
name = "runtime_task"
harness = false
135 changes: 135 additions & 0 deletions src/llm-coding-tools-agents/benches/runtime_task.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
//! Benchmarks for [`AgentRuntime`] task-delegation cache lookups.
//!
//! Measures the cost of [`AgentRuntime::allowed_tools`],
//! [`AgentRuntime::summarize_callable_targets`], and
//! [`AgentRuntime::can_delegate_to`] across varying agent counts.

use ahash::AHashMap;
use core::hint::black_box;
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
use indexmap::IndexMap;
use llm_coding_tools_agents::{
AgentCatalog, AgentConfig, AgentMode, AgentRuntimeBuilder, AgentToolSettings, PermissionRule,
};
use llm_coding_tools_core::permissions::PermissionAction;
use llm_coding_tools_core::tool_metadata::{read as read_meta, task as task_meta};

/// Build a minimal [`AgentConfig`] for benchmark fixtures.
///
/// `permission` controls tool-access rules; all other fields are filled
/// with placeholder values suitable for performance measurement only.
fn build_agent(
name: &str,
mode: AgentMode,
permission: IndexMap<String, PermissionRule>,
) -> AgentConfig {
AgentConfig {
name: name.into(),
mode,
description: format!("{name} description").into(),
model: None,
hidden: false,
temperature: None,
top_p: None,
permission,
options: AHashMap::new(),
tool_settings: AgentToolSettings::default(),
prompt: Default::default(),
}
}

/// Create a permission map that denies all tools by default, but allows
/// pattern-matched delegation to agents named `review-*` or `worker-*`
/// via the task tool, and blanket-allows the read tool.
fn patterned_task_permission() -> IndexMap<String, PermissionRule> {
let mut patterns = IndexMap::new();
patterns.insert("*".to_string(), PermissionAction::Deny);
patterns.insert("review-*".to_string(), PermissionAction::Allow);
patterns.insert("worker-*".to_string(), PermissionAction::Allow);

IndexMap::from([
(task_meta::NAME.into(), PermissionRule::Pattern(patterns)),
(
read_meta::NAME.into(),
PermissionRule::Action(PermissionAction::Allow),
),
])
}

/// Build an [`AgentRuntime`] with one `caller` primary agent and
/// `agent_count` subordinate agents.
///
/// Subordinate names cycle through `review-NNN`, `worker-NNN`, and
/// `misc-NNN` prefixes. Every 11th subordinate is a primary-mode agent;
/// the rest are subagents.
fn build_runtime(agent_count: usize) -> llm_coding_tools_agents::AgentRuntime {
let mut agents = Vec::with_capacity(agent_count + 1);
agents.push(build_agent(
"caller",
AgentMode::Primary,
patterned_task_permission(),
));

for idx in 0..agent_count {
let name = match idx % 3 {
0 => format!("review-{idx:03}"),
1 => format!("worker-{idx:03}"),
_ => format!("misc-{idx:03}"),
};
let mode = if idx % 11 == 0 {
AgentMode::Primary
} else {
AgentMode::Subagent
};
agents.push(build_agent(&name, mode, IndexMap::new()));
}

AgentRuntimeBuilder::new()
.catalog(AgentCatalog::from_entries(agents))
.build()
}

/// Benchmark cached delegation queries against runtimes of 16, 64, and 256 agents.
///
/// Measures four operations:
/// - **allowed_tools** – full tool-set resolution for the `caller` agent.
/// - **summaries** – callable-target summary strings for `caller`.
/// - **can_delegate_hit** – pattern-match hit (`caller` → `review-003`).
/// - **can_delegate_miss** – pattern-match miss (`caller` → `misc-002`).
fn bench_runtime_task_caches(c: &mut Criterion) {
Comment thread
coderabbitai[bot] marked this conversation as resolved.
let mut group = c.benchmark_group("runtime/task_caches");

for agent_count in [16_usize, 64, 256] {
let runtime = build_runtime(agent_count);
group.throughput(Throughput::Elements(1));

group.bench_with_input(
BenchmarkId::new("allowed_tools", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.allowed_tools("caller"))),
);

group.bench_with_input(
BenchmarkId::new("summaries", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.summarize_callable_targets("caller"))),
);

group.bench_with_input(
BenchmarkId::new("can_delegate_hit", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.can_delegate_to("caller", "review-003"))),
);

group.bench_with_input(
BenchmarkId::new("can_delegate_miss", agent_count),
&runtime,
|b, runtime| b.iter(|| black_box(runtime.can_delegate_to("caller", "misc-002"))),
);
}

group.finish();
}

criterion_group!(benches, bench_runtime_task_caches);
criterion_main!(benches);
6 changes: 3 additions & 3 deletions src/llm-coding-tools-agents/src/extensions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,11 @@ pub trait RulesetExt {
/// let ruleset = Ruleset::from_permission_config(&config);
/// assert!(ruleset.is_allowed("bash", "*"));
/// ```
fn from_permission_config<'a>(config: &'a IndexMap<String, PermissionRule>) -> Ruleset<'a>;
fn from_permission_config(config: &IndexMap<String, PermissionRule>) -> Ruleset;
}

impl RulesetExt for Ruleset<'_> {
fn from_permission_config<'a>(config: &'a IndexMap<String, PermissionRule>) -> Ruleset<'a> {
impl RulesetExt for Ruleset {
fn from_permission_config(config: &IndexMap<String, PermissionRule>) -> Ruleset {
let mut ruleset = Ruleset::with_capacity(config.len() * 2);

for (key, rule) in config {
Expand Down
37 changes: 36 additions & 1 deletion src/llm-coding-tools-agents/src/runtime/builder.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,12 @@ mod tests {
use super::AgentRuntimeBuilder;
use crate::runtime::tool_catalog::{default_tools, ToolCatalogEntry, ToolCatalogKind};
use crate::runtime::AgentDefaults;
use crate::{AgentCatalog, AgentConfig, AgentMode, AgentToolSettings};
use crate::{AgentCatalog, AgentConfig, AgentMode, AgentToolSettings, PermissionRule};
use indexmap::IndexMap;
use llm_coding_tools_core::permissions::PermissionAction;
use llm_coding_tools_core::tool_metadata::{glob as glob_meta, read as read_meta};
use llm_coding_tools_core::TaskSettings;
use std::sync::Arc;

fn sample_config(name: &str, model: Option<&str>) -> AgentConfig {
AgentConfig {
Expand Down Expand Up @@ -147,4 +150,36 @@ mod tests {
assert_eq!(runtime.task_settings(), TaskSettings::default());
assert_eq!(runtime.tools(), default_tools().as_slice());
}

#[test]
fn builder_caches_permission_rulesets() {
let runtime = AgentRuntimeBuilder::new()
.catalog(AgentCatalog::from_entries([AgentConfig {
name: "planner".into(),
mode: AgentMode::Subagent,
description: "planner description".into(),
model: None,
hidden: false,
temperature: None,
top_p: None,
permission: IndexMap::from([(
read_meta::NAME.into(),
PermissionRule::Action(PermissionAction::Allow),
)]),
options: Default::default(),
tool_settings: AgentToolSettings::default(),
prompt: Default::default(),
}]))
.build();

let first = runtime
.permission_ruleset("planner")
.expect("cached ruleset should exist");
let second = runtime
.permission_ruleset("planner")
.expect("cached ruleset should exist");

assert!(Arc::ptr_eq(&first, &second));
assert!(first.is_allowed(read_meta::NAME, "*"));
}
}
90 changes: 86 additions & 4 deletions src/llm-coding-tools-agents/src/runtime/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
//! - [`AgentRuntime`] — Container for loaded agents, defaults, Task settings, and tools.
//! - [`AgentDefaults`] — Fallback settings when an agent doesn't specify them.

use super::task::resolve_allowed_tools;
use super::task::{build_runtime_task_caches, TaskTargetSummary};
use super::tool_catalog::ToolCatalogEntry;
use crate::AgentCatalog;
use crate::{AgentCatalog, RulesetExt};
use ahash::AHashMap;
use llm_coding_tools_core::permissions::Ruleset;
use llm_coding_tools_core::TaskSettings;
use std::sync::Arc;

/// Default settings used when an agent doesn't specify them.
#[derive(Debug, Clone, Default, PartialEq)]
Expand Down Expand Up @@ -40,6 +43,9 @@ pub struct AgentRuntime {
defaults: AgentDefaults,
task_settings: TaskSettings,
tools: Vec<ToolCatalogEntry>,
permission_rulesets: AHashMap<String, Arc<Ruleset>>,
allowed_tools_by_caller: AHashMap<String, Vec<ToolCatalogEntry>>,
callable_target_summaries_by_caller: AHashMap<String, Vec<TaskTargetSummary>>,
}

impl AgentRuntime {
Expand All @@ -50,11 +56,26 @@ impl AgentRuntime {
task_settings: TaskSettings,
tools: Vec<ToolCatalogEntry>,
) -> Self {
let permission_rulesets = catalog
.iter()
.map(|agent| {
(
agent.name.to_string(),
Arc::new(Ruleset::from_permission_config(&agent.permission)),
)
})
.collect();
let (allowed_tools_by_caller, callable_target_summaries_by_caller) =
build_runtime_task_caches(&catalog, &permission_rulesets, &tools);

Self {
catalog,
defaults,
task_settings,
tools,
permission_rulesets,
allowed_tools_by_caller,
callable_target_summaries_by_caller,
}
}

Expand Down Expand Up @@ -82,13 +103,74 @@ impl AgentRuntime {
&self.tools
}

/// Returns the cached permission ruleset for the named caller.
///
/// The returned [`Arc`] is cheap to clone and reuses the ruleset built when
/// the runtime was constructed.
#[inline]
pub fn permission_ruleset(&self, caller_name: &str) -> Option<Arc<Ruleset>> {
self.permission_rulesets.get(caller_name).cloned()
}

/// Returns the tool entries exposed to the named caller.
///
/// Most tools use the standard wildcard permission check (`permission -> "*"`).
/// `task` is only included when at least one `mode: all` or `mode: subagent`
/// target remains callable after applying `permission.task`.
#[inline]
pub fn allowed_tools(&self, caller_name: &str) -> Vec<ToolCatalogEntry> {
resolve_allowed_tools(self, caller_name)
pub fn allowed_tools(&self, caller_name: &str) -> &[ToolCatalogEntry] {
self.allowed_tools_by_caller
.get(caller_name)
.map(Vec::as_slice)
.unwrap_or(&[])
}

/// Returns stable summaries for every agent the named caller may delegate to via Task.
///
/// Only agents with [`AgentMode::All`] or [`AgentMode::Subagent`] are
/// considered. When the caller defines `permission.task`, targets are
/// filtered by those rules; otherwise all non-primary agents are included.
/// Results are sorted alphabetically by target name.
///
/// # Arguments
///
/// * `caller_name` - Name of the agent that wants to delegate.
///
/// # Returns
///
/// A [`TaskTargetSummary`] per callable target. Empty if `caller_name`
/// is not in the catalog or no targets survive permission filtering.
///
/// [`AgentMode::All`]: crate::AgentMode::All
/// [`AgentMode::Subagent`]: crate::AgentMode::Subagent
#[inline]
pub fn summarize_callable_targets(&self, caller_name: &str) -> &[TaskTargetSummary] {
self.callable_target_summaries_by_caller
.get(caller_name)
.map(Vec::as_slice)
.unwrap_or(&[])
}

/// Returns whether the named caller may delegate to the given target.
///
/// Looks up the caller's filtered callable-target list and performs a
/// binary search for `target_name`.
///
/// # Arguments
/// - `caller_name`: Name of the agent that would originate the delegation.
/// - `target_name`: Name of the candidate delegate target.
///
/// # Returns
/// - `true` if `target_name` appears in the caller's permitted target list.
/// - `false` if the caller is not in the catalog or the target is absent.
#[inline]
pub fn can_delegate_to(&self, caller_name: &str, target_name: &str) -> bool {
self.callable_target_summaries_by_caller
.get(caller_name)
.is_some_and(|summaries| {
summaries
.binary_search_by(|summary| summary.name.as_ref().cmp(target_name))
.is_ok()
})
}
}
Loading