diff --git a/Cargo.lock b/Cargo.lock index a4e85f5..8a86aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "agent-spec" +version = "0.1.0" +dependencies = [ + "anyhow", + "kdl", + "serde", + "serde_json", + "tempfile", + "toml", +] + [[package]] name = "anstream" version = "1.0.0" @@ -514,6 +526,7 @@ dependencies = [ name = "st2" version = "0.1.0" dependencies = [ + "agent-spec", "anyhow", "clap", "clap_complete", diff --git a/Cargo.toml b/Cargo.toml index 2814faf..451d388 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,10 @@ +# The root stays a real package (not a virtual manifest): `flake.nix` reads +# `package.version` out of this file as the single source of truth for the +# build, and a virtual root has no `[package]` to read. +[workspace] +members = ["crates/agent-spec"] +default-members = [".", "crates/agent-spec"] + [package] name = "st2" version = "0.1.0" @@ -16,6 +23,7 @@ path = "src/lib.rs" [dependencies] # Sync by design: the runner is I/O-light (shell-outs to `pty` plus a # folder watch + a sleep), so no tokio. Deps grow per milestone; M0 needs only parse + CLI. +agent-spec = { path = "crates/agent-spec" } anyhow = "1" clap = { version = "4", features = ["derive"] } clap_complete = "4" diff --git a/crates/agent-spec/Cargo.toml b/crates/agent-spec/Cargo.toml new file mode 100644 index 0000000..9b70955 --- /dev/null +++ b/crates/agent-spec/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "agent-spec" +version = "0.1.0" +edition = "2024" +description = "Parse a catalog of rendered agent specs (KDL/TOML/JSON) into the runner-normative agent job model." +license = "MIT" + +[dependencies] +anyhow = "1" +kdl = "6" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +toml = "0.9" + +[dev-dependencies] +tempfile = "3" diff --git a/src/discovery.rs b/crates/agent-spec/src/discovery.rs similarity index 85% rename from src/discovery.rs rename to crates/agent-spec/src/discovery.rs index 994cd7b..139a083 100644 --- a/src/discovery.rs +++ b/crates/agent-spec/src/discovery.rs @@ -92,10 +92,40 @@ fn collect_spec_files(root: &Path, dir: &Path, acc: &mut Vec) { } } +/// What a declaration literally *says*, before lowering normalizes it away. +/// +/// Lowering is lossy by design: a typo'd `type = "srvice"` becomes `JobType::Service`, and an +/// identity omitted from the content is filled in from the path. Both are invisible in the resolved +/// [`AgentSpec`], so a linter that wants to fault them has to see the declared form. This is that +/// view — deliberately narrow, so the permissive on-disk shape itself stays private and is free to +/// gain fields without breaking readers. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Declared { + /// `identity` as written in the file. `None` when the file relies on [`path_defaults`]. + pub identity: Option, + /// `type` as written, before it is normalized to `JobType::Service`. `None` when unset. + pub job_type: Option, +} + +/// Read the declared (pre-lowering) values of every agent in a file — one per `agent` node for KDL, +/// 0-or-1 for TOML/JSON, empty for a non-spec extension. +/// +/// One entry per parsed node, *including* nodes [`discover`] skips as non-specs, so this is not +/// positionally paired with that file's [`Discovered::specs`]. +pub fn parse_declared(path: &Path) -> anyhow::Result> { + Ok(parse_raw_file(path)? + .into_iter() + .map(|raw| Declared { + identity: raw.identity, + job_type: raw.job_type, + }) + .collect()) +} + /// Parse a spec file into its raw (pre-resolution) shape — one per `agent` node for KDL, 0-or-1 for -/// TOML/JSON. Non-spec extensions yield an empty vec. Shared by discovery and `validate` (which needs -/// the *raw* `type` string before it is normalized away, to catch a typo'd `type = "srvice"`). -pub(crate) fn parse_raw_file(path: &Path) -> anyhow::Result> { +/// TOML/JSON. Non-spec extensions yield an empty vec. Shared by discovery and [`parse_declared`] +/// (which exposes the *raw* `type` and `identity` before normalization, without leaking [`RawSpec`]). +fn parse_raw_file(path: &Path) -> anyhow::Result> { let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); let text = fs::read_to_string(path)?; Ok(match ext { diff --git a/src/kdl_format.rs b/crates/agent-spec/src/kdl_format.rs similarity index 100% rename from src/kdl_format.rs rename to crates/agent-spec/src/kdl_format.rs diff --git a/crates/agent-spec/src/lib.rs b/crates/agent-spec/src/lib.rs new file mode 100644 index 0000000..b762676 --- /dev/null +++ b/crates/agent-spec/src/lib.rs @@ -0,0 +1,28 @@ +//! agent-spec — read a catalog of rendered agent declarations. +//! +//! One agent is one declarative file, Nomad-style: the agent is the job and its `pty`/`exec` blocks +//! are the tasks. This crate owns the two halves any reader of that catalog needs and nothing else: +//! +//! - [`spec`] — the runner-normative model a declaration lowers to ([`AgentSpec`], [`Task`], …). +//! - [`discovery`] — the catalog walk: parse every `*.{kdl,toml,json}` that looks like a +//! declaration, and resolve each one's `identity`/`host` with the catalog's precedence rule +//! (content wins, the path supplies defaults, a mismatch is a warning). +//! +//! KDL is the canonical on-disk format; TOML and JSON lower to the same model. The KDL parser is a +//! private implementation detail — [`discovery`] is the only supported entry point, so every reader +//! resolves identity and host the same way rather than re-deriving it from filenames. +//! +//! st2 consumes this crate, which is what keeps it a reference implementation rather than a copy: +//! a second reader (a TUI, a linter) sees exactly the fields the runner sees, including the ones +//! the runner's roster JSON does not carry (`supervisor`, `role`, `workspace`, `host`). +//! +//! Render-only fields (`harness`, `model`, `persona`, `permissions`, `transport`, `strategy`, +//! `meta{}`) are read by the render layer and deliberately dropped here — that is what keeps a +//! consumer render-agnostic. + +pub mod discovery; +mod kdl_format; +pub mod spec; + +pub use discovery::{Declared, Discovered, SpecError, discover, parse_declared, path_defaults}; +pub use spec::{AgentSpec, JobType, Restart, RestartMode, Task, TaskKind, parse_duration}; diff --git a/src/spec.rs b/crates/agent-spec/src/spec.rs similarity index 100% rename from src/spec.rs rename to crates/agent-spec/src/spec.rs diff --git a/tests/discovery.rs b/crates/agent-spec/tests/discovery.rs similarity index 97% rename from tests/discovery.rs rename to crates/agent-spec/tests/discovery.rs index cf07a38..31463b8 100644 --- a/tests/discovery.rs +++ b/crates/agent-spec/tests/discovery.rs @@ -1,15 +1,15 @@ //! M1 correctness net: discovery + lowering of VRS `agent.kdl` jobs (spec.md §1–2, §4). //! //! Builds throwaway catalog folders, writes real job files (KDL/TOML/JSON, services), and -//! asserts st2 lowers them per the spec: `pty`/`exec` task split, `restart{}`, `type`, `workspace`, +//! asserts they lower per the spec: `pty`/`exec` task split, `restart{}`, `type`, `workspace`, //! `supervisor`; render-only fields ignored; content/path precedence; malformed → error, not halt. use std::fs; use std::path::Path; use std::time::Duration; -use st2::spec::TaskKind; -use st2::{AgentSpec, JobType, discover}; +use agent_spec::spec::TaskKind; +use agent_spec::{AgentSpec, JobType, discover}; fn write(root: &Path, rel: &str, contents: &str) { let path = root.join(rel); @@ -91,7 +91,7 @@ fn parses_full_kdl_service_job() { assert_eq!(r.attempts, 5); assert_eq!(r.interval, Duration::from_secs(90)); assert_eq!(r.delay, Duration::from_secs(5)); - assert_eq!(r.mode, st2::RestartMode::Fail); + assert_eq!(r.mode, agent_spec::RestartMode::Fail); // tasks: pty "agent" + exec "ding" (sorted by name) assert_eq!(s.tasks.len(), 2); @@ -213,7 +213,10 @@ command = "st2 ding hetz.fetcher" let s = &found.specs[0]; assert_eq!(s.identity, "fetcher"); assert_eq!(s.job_type, JobType::Service); - assert_eq!(s.restart.clone().unwrap().mode, st2::RestartMode::Delay); + assert_eq!( + s.restart.clone().unwrap().mode, + agent_spec::RestartMode::Delay + ); assert_eq!(s.tasks.len(), 2); assert_eq!( s.tasks.iter().find(|t| t.name == "agent").unwrap().kind, diff --git a/flake.nix b/flake.nix index 87d4227..b2b332a 100644 --- a/flake.nix +++ b/flake.nix @@ -87,15 +87,21 @@ --fish completions-fish ''; - # Run the hermetic unit tests plus the real lifecycle-hook integration - # test. The remaining integration tests assume facilities the Nix build - # sandbox deliberately lacks: `/usr/bin/git` on a hardcoded `PATH`, - # live PTY backends, or a systemd `--user` manager. They remain native - # gates, while the flake proves that its own packaged hooks execute. + # Run the hermetic unit tests plus agent-spec's discovery test and the + # real lifecycle-hook integration tests. The remaining root integration + # tests assume facilities the Nix build sandbox deliberately lacks: + # `/usr/bin/git` on a hardcoded `PATH`, live PTY backends, or a systemd + # `--user` manager. They remain native gates, while the flake proves + # that its parser and packaged hooks execute. + # `--workspace` because the root is a real package: without it cargo + # selects only `st2` and silently skips the `agent-spec` crate. cargoTestFlags = [ + "--workspace" "--lib" "--bins" "--test" + "discovery" + "--test" "codex_hooks" "--test" "hooks" diff --git a/src/eval_run.rs b/src/eval_run.rs index 3395c80..7a61fb0 100644 --- a/src/eval_run.rs +++ b/src/eval_run.rs @@ -16,7 +16,7 @@ use crate::expand::expand_catalog; use crate::flapping::FlappingCap; use crate::reconcile::reconcile; use crate::run::{Runner, SystemRunner, UpReport, detect_host, execute}; -use crate::spec::{AgentSpec, JobType, Task, TaskKind}; +use agent_spec::spec::{AgentSpec, JobType, Task, TaskKind}; macro_rules! eval_log { ($($arg:tt)*) => { diff --git a/src/eval_spec.rs b/src/eval_spec.rs index 67e56d0..84d4c65 100644 --- a/src/eval_spec.rs +++ b/src/eval_spec.rs @@ -15,7 +15,7 @@ use std::time::Duration; use kdl::{KdlDocument, KdlNode, KdlValue}; -use crate::spec::{Restart, RestartMode, parse_duration}; +use agent_spec::spec::{Restart, RestartMode, parse_duration}; /// A parsed st2 spec: a base team (`st2 up` boots this) plus an optional `eval` (`st2 eval` runs it). #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/flapping.rs b/src/flapping.rs index a52aefd..fa873c6 100644 --- a/src/flapping.rs +++ b/src/flapping.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::time::Instant; -use crate::spec::{Restart, RestartMode}; +use agent_spec::spec::{Restart, RestartMode}; /// What to do with a would-be (re)launch this pass. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/hooks.rs b/src/hooks.rs index 56905cd..522711b 100644 --- a/src/hooks.rs +++ b/src/hooks.rs @@ -125,14 +125,14 @@ pub fn versioned_hooks_dir_at(root: &Path) -> PathBuf { } /// Whether this host has a Codex agent whose rendered settings and next launch need this hook set. -pub fn required_by_codex(specs: &[crate::spec::AgentSpec], this_host: &str) -> bool { +pub fn required_by_codex(specs: &[agent_spec::spec::AgentSpec], this_host: &str) -> bool { specs .iter() .any(|spec| required_by_codex_agent(spec, this_host)) } /// Whether one local declaration owns a Codex agent task. -pub fn required_by_codex_agent(spec: &crate::spec::AgentSpec, this_host: &str) -> bool { +pub fn required_by_codex_agent(spec: &agent_spec::spec::AgentSpec, this_host: &str) -> bool { spec.host.as_deref().is_none_or(|host| host == this_host) && spec.tasks.iter().any(|task| { task.name == "agent" && task.command.as_deref().is_some_and(command_invokes_codex) diff --git a/src/lib.rs b/src/lib.rs index 9def522..6bddfda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,7 +8,6 @@ pub mod agents; pub mod compile_agent; pub mod context; pub mod ding; -pub mod discovery; pub mod eval_run; pub mod eval_spec; pub mod exec_backend; @@ -17,7 +16,6 @@ pub mod flapping; pub mod hooks; pub mod host_lock; pub mod isolate; -mod kdl_format; pub mod materialize; pub mod message; pub mod pretrust; @@ -25,13 +23,20 @@ pub mod reconcile; pub mod resource; pub mod run; pub mod service; -pub mod spec; pub mod status; pub mod validate; pub mod version; mod watch; -pub use discovery::{Discovered, SpecError, discover}; +// The declaration model and the catalog walk live in the `agent-spec` crate, so st2 and any other +// reader of the same catalog share one implementation. Re-exported under their original paths: +// `st2::spec::…` / `st2::discovery::…` keep working for the binary and the test suite. +pub use agent_spec::{discovery, spec}; + +pub use agent_spec::discovery::{Discovered, SpecError, discover}; +pub use agent_spec::spec::{ + AgentSpec, JobType, Restart, RestartMode, Task, TaskKind, parse_duration, +}; pub use exec_backend::ExecBackend; pub use expand::{expand_env, expand_vars}; pub use flapping::FlappingCap; @@ -41,4 +46,3 @@ pub use run::{ PtyCli, Runner, SystemRunner, UpReport, detect_host, down, down_specs, exec_state_dir, execute, up_loop, up_loop_specs, up_once, up_once_specs, }; -pub use spec::{AgentSpec, JobType, Restart, RestartMode, Task, TaskKind, parse_duration}; diff --git a/src/materialize.rs b/src/materialize.rs index 65083b2..89f021d 100644 --- a/src/materialize.rs +++ b/src/materialize.rs @@ -11,7 +11,7 @@ use std::process::Command; use anyhow::{Context, Result}; use kdl::{KdlDocument, KdlNode}; -use crate::spec::AgentSpec; +use agent_spec::spec::AgentSpec; #[derive(Debug, Clone, PartialEq, Eq)] pub enum RenderOp { diff --git a/src/reconcile.rs b/src/reconcile.rs index 1b08136..fd08594 100644 --- a/src/reconcile.rs +++ b/src/reconcile.rs @@ -11,7 +11,7 @@ use std::collections::BTreeMap; use std::collections::HashMap; -use crate::spec::{AgentSpec, TaskKind}; +use agent_spec::spec::{AgentSpec, TaskKind}; /// ACTUAL state: one running/known task as st2 observes it (unioned across backends). #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/run.rs b/src/run.rs index 8ef9976..ac0f281 100644 --- a/src/run.rs +++ b/src/run.rs @@ -27,7 +27,7 @@ use crate::exec_backend::ExecBackend; use crate::flapping::FlappingCap; use crate::message; use crate::reconcile::{ReconcilePlan, Session, TaskTarget}; -use crate::spec::TaskKind; +use agent_spec::spec::TaskKind; const PTY_LIST_TIMEOUT: Duration = Duration::from_secs(2); const PTY_DAEMON_SHUTDOWN_WAIT: Duration = Duration::from_secs(6); @@ -896,7 +896,7 @@ pub fn up_once(root: &Path, this_host: &str, runner: &dyn Runner) -> anyhow::Res /// once (the carried cap rate-limits a same-episode second respawn; the carried debounce absorbs a /// transient `pty list` misread of a healthy seat). pub fn reconcile_pass_specs( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], this_host: &str, runner: &dyn Runner, cap: &mut FlappingCap, @@ -921,7 +921,7 @@ pub fn reconcile_pass_specs( /// process can exit between two `pty list` calls, be reaped by the second call, then look like a /// vanished crash on the next tick. pub(crate) fn reconcile_pass_specs_with_sessions( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], sessions: &[Session], this_host: &str, runner: &dyn Runner, @@ -940,7 +940,7 @@ pub(crate) fn reconcile_pass_specs_with_sessions( /// One reconcile pass over an in-memory spec team (`st2 up --once`). Throwaway cap+debounce /// (a single pass has no flicker history); never `Err` — failures collect in `report.errors`. pub fn up_once_specs( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], this_host: &str, runner: &dyn Runner, ) -> UpReport { @@ -960,7 +960,7 @@ pub fn up_once_specs( /// (a spec is one static file — no folder to watch; edit + restart to change it). `root` roots /// `$CATALOG` + crash-loop surfacing. Runs until SIGINT/SIGTERM. pub fn up_loop_specs( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], root: &Path, this_host: &str, runner: &dyn Runner, @@ -1027,7 +1027,7 @@ pub fn down(root: &Path, this_host: &str, runner: &dyn Runner) -> anyhow::Result /// Sessions persist across an `st2 up` supervisor exit (nomad-decoupled), so this is how you actually /// stop them. `specs` are the already-resolved [`AgentSpec`]s (from `spec_to_agent_specs`). pub fn down_specs( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], this_host: &str, runner: &dyn Runner, ) -> anyhow::Result { @@ -1040,7 +1040,7 @@ pub fn down_specs( /// derived identically to how reconcile spawns them (explicit `task.id`, else `.`), so /// the catalog `down` and the spec `down_specs` tear down exactly what `up`/`up_*_specs` launched. fn teardown_specs( - specs: &[crate::spec::AgentSpec], + specs: &[agent_spec::spec::AgentSpec], this_host: &str, runner: &dyn Runner, report: &mut UpReport, @@ -1238,7 +1238,7 @@ pub fn detect_host() -> String { #[cfg(test)] mod tests { use super::*; - use crate::spec::TaskKind; + use agent_spec::spec::TaskKind; use std::collections::BTreeMap; use std::ffi::OsStr; @@ -1312,7 +1312,7 @@ mod tests { // destructively GC/relaunch a HEALTHY agent; a stable death must still be reaped ────────────── use crate::reconcile::Launch; - use crate::spec::{AgentSpec, JobType}; + use agent_spec::spec::{AgentSpec, JobType}; fn sess(id: &str, alive: bool) -> Session { Session { diff --git a/src/validate.rs b/src/validate.rs index 646a998..6e28405 100644 --- a/src/validate.rs +++ b/src/validate.rs @@ -18,8 +18,8 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; -use crate::discovery::{discover, parse_raw_file, path_defaults}; -use crate::spec::{AgentSpec, JobType}; +use agent_spec::discovery::{discover, parse_declared, path_defaults}; +use agent_spec::spec::{AgentSpec, JobType}; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Severity { @@ -140,7 +140,7 @@ fn validate_scoped(root: &Path, this_host: Option<&str>) -> Report { files.sort(); files.dedup(); for f in files { - if let Ok(raws) = parse_raw_file(f) { + if let Ok(raws) = parse_declared(f) { for raw in raws { if let Some(t) = &raw.job_type && t != "service"