From 8640f1a871aef0bfca2615b4dfe1c10f92c64fda Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:28:38 +0300 Subject: [PATCH 01/55] chore(deps): add tinyruntime submodule Add the tinyruntime repository as a new Git submodule under vendor/tinyruntime, pinning it to commit 02418aa86. This makes the runtime dependency available for local development and ensures all contributors use the same version. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .gitmodules | 3 +++ vendor/tinyruntime | 1 + 2 files changed, 4 insertions(+) create mode 160000 vendor/tinyruntime diff --git a/.gitmodules b/.gitmodules index 77e5f3d4d8..63a25ba42e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ path = vendor/tinyhosts url = https://github.com/tinyhumansai/tinyhosts.git branch = main +[submodule "vendor/tinyruntime"] + path = vendor/tinyruntime + url = https://github.com/tinyhumansai/tinyruntime.git diff --git a/vendor/tinyruntime b/vendor/tinyruntime new file mode 160000 index 0000000000..02418aa861 --- /dev/null +++ b/vendor/tinyruntime @@ -0,0 +1 @@ +Subproject commit 02418aa8614c1c859f4ac7407c4757f9b296f68e From 01b8323711b7cab0803e924ef2d65f98a616358d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:28:50 +0300 Subject: [PATCH 02/55] chore(deps): add tinyruntime-bus dependency Adds the tinyruntime-bus crate as a workspace dependency, pointing to the vendor/tinyruntime submodule. The crate provides only the contract types for the TinyBus runtime router, deliberately avoiding the router itself and its heavy dependency tree. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 5c078d41b5..e0bc084590 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -244,6 +244,19 @@ tinycortex-api = { path = "vendor/tinycortex/api" } # rather than the one pinned inside the tinymemory submodule. # # After cloning: `git submodule update --init --recursive vendor/tinymemory`. +# tinyruntime — the runtime router +# (https://github.com/tinyhumansai/tinyruntime). Only the *contract* crate is +# taken here, never the router itself: the router ships as a loadable TinyBus +# module, and a host that also compiled it would carry the download pipeline, the +# worker pool, and their dependency trees for nothing. +# +# `tinyruntime-bus` is deliberately dependency-light — `serde` and `serde_json` +# and nothing else — which is what makes naming the payload types cost this +# manifest almost nothing. The module's own CI asserts it stays that way. +# +# After cloning: `git submodule update --init --recursive vendor/tinyruntime`. +tinyruntime-bus = { path = "vendor/tinyruntime/crates/tinyruntime-bus" } + tinymemory = { path = "vendor/tinymemory/crates/tinymemory" } tinymemory-api = { path = "vendor/tinymemory/crates/tinymemory-api" } tinymemory-tinycortex = { path = "vendor/tinymemory/crates/tinymemory-tinycortex" } From aebac0d65c3f4c8af74fe47727d9e728a0f3b471 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:29:20 +0300 Subject: [PATCH 03/55] feat(registry): add tinyruntime module records Add three new module records for the tinyruntime system: the runtime router itself and its Node.js and Python provider modules. These are declared lazy because a host that never runs a skill or flow step should not pay the cost of downloading and loading runtime support. The assets arrays are deliberately empty until the first published releases are cut. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/registry.rs | 81 ++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/src/openhuman/modules/registry.rs b/src/openhuman/modules/registry.rs index bfda832f29..cc84bd7acb 100644 --- a/src/openhuman/modules/registry.rs +++ b/src/openhuman/modules/registry.rs @@ -421,8 +421,87 @@ const TINYVOICE: ModuleRecord = ModuleRecord { load: LoadPolicy::Lazy, }; +/// The `tinyruntime` module: the runtime router. +/// +/// Resolves a language runtime, installs one when the host has none, reuses one +/// when it does, and runs code on a bounded pool of warm interpreter processes. +/// It is a router: on its own it knows no languages, and it routes to the two +/// provider records below. +/// +/// Lazy, because a host that never runs a skill, a flow step, or a `node_exec` +/// should not pay a download and a `dlopen` for the ability to. +/// +/// # No pinned assets yet +/// +/// `assets` is deliberately empty: this build pins no published release. The +/// module still loads from a developer build named by `modules.local` or from +/// the module search path (`OPENHUMAN_MODULE_PATH`), which is how it is +/// exercised today. A download attempt reports that no artifact exists for this +/// platform, which is accurate. +/// +/// When the first release is cut, take the digests verbatim from that release's +/// `checksum.toml` — never from a local build, which would agree with itself no +/// matter what was served. +const TINYRUNTIME: ModuleRecord = ModuleRecord { + id: "tinyruntime", + description: "Language runtime resolution, installation, and pooled execution", + bus_name: "ai.tinyhumans.runtime.Runtime", + object_path: "/ai/tinyhumans/runtime/Runtime", + version: "0.1.0", + release_url: "https://github.com/tinyhumansai/tinyruntime/releases/tag/v0.1.0", + assets: &[], + load: LoadPolicy::Lazy, +}; + +/// The `tinyruntime-nodejs` module: the Node.js half of the router's knowledge. +/// +/// Answers which host interpreters count, which archive nodejs.org publishes for +/// this machine, where the binaries land, and what a warm Node worker is. It +/// installs nothing itself. +/// +/// Lazy, and loaded by the same call that loads the router: a language is only +/// worth its `dlopen` when something asks for that language. +/// +/// See [`TINYRUNTIME`] on why `assets` is empty. +const TINYRUNTIME_NODEJS: ModuleRecord = ModuleRecord { + id: "tinyruntime-nodejs", + description: "Node.js runtime provider for tinyruntime", + bus_name: "ai.tinyhumans.runtime.nodejs.Provider", + object_path: "/ai/tinyhumans/runtime/Provider", + version: "0.1.0", + release_url: "https://github.com/tinyhumansai/tinyruntime-nodejs/releases/tag/v0.1.0", + assets: &[], + load: LoadPolicy::Lazy, +}; + +/// The `tinyruntime-python` module: the Python half of the router's knowledge. +/// +/// Answers which host interpreters count, which standalone build to install, and +/// what a warm Python worker is. It installs nothing itself. +/// +/// See [`TINYRUNTIME`] on why `assets` is empty. +const TINYRUNTIME_PYTHON: ModuleRecord = ModuleRecord { + id: "tinyruntime-python", + description: "Python runtime provider for tinyruntime", + bus_name: "ai.tinyhumans.runtime.python.Provider", + object_path: "/ai/tinyhumans/runtime/Provider", + version: "0.1.0", + release_url: "https://github.com/tinyhumansai/tinyruntime-python/releases/tag/v0.1.0", + assets: &[], + load: LoadPolicy::Lazy, +}; + /// Every module this build can load. -pub const ALL: &[ModuleRecord] = &[TINYDOCS, TINYWALLET, TINYMEMORY, TINYJUICE, TINYVOICE]; +pub const ALL: &[ModuleRecord] = &[ + TINYDOCS, + TINYWALLET, + TINYMEMORY, + TINYJUICE, + TINYVOICE, + TINYRUNTIME, + TINYRUNTIME_NODEJS, + TINYRUNTIME_PYTHON, +]; /// The record for `id`, if this build knows it. #[must_use] From 44a81f071ecfdd2ab8d04e948de5a0618ededac4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:33:27 +0300 Subject: [PATCH 04/55] fix(registry): give each runtime provider its own object path The Node.js and Python runtime providers were sharing the same D-Bus object path as the generic tinyruntime provider, which is invalid because two modules cannot claim the same bus name and tinybus derives the path from the name. Each provider now uses a unique path that includes its language identifier. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/registry.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/openhuman/modules/registry.rs b/src/openhuman/modules/registry.rs index cc84bd7acb..83c0c05ebd 100644 --- a/src/openhuman/modules/registry.rs +++ b/src/openhuman/modules/registry.rs @@ -459,6 +459,10 @@ const TINYRUNTIME: ModuleRecord = ModuleRecord { /// this machine, where the binaries land, and what a warm Node worker is. It /// installs nothing itself. /// +/// It implements the shared `ai.tinyhumans.runtime.Provider` interface but +/// serves at its own object path, because two modules cannot claim one bus name +/// and tinybus derives the path from the name. +/// /// Lazy, and loaded by the same call that loads the router: a language is only /// worth its `dlopen` when something asks for that language. /// @@ -467,7 +471,7 @@ const TINYRUNTIME_NODEJS: ModuleRecord = ModuleRecord { id: "tinyruntime-nodejs", description: "Node.js runtime provider for tinyruntime", bus_name: "ai.tinyhumans.runtime.nodejs.Provider", - object_path: "/ai/tinyhumans/runtime/Provider", + object_path: "/ai/tinyhumans/runtime/nodejs/Provider", version: "0.1.0", release_url: "https://github.com/tinyhumansai/tinyruntime-nodejs/releases/tag/v0.1.0", assets: &[], @@ -484,7 +488,7 @@ const TINYRUNTIME_PYTHON: ModuleRecord = ModuleRecord { id: "tinyruntime-python", description: "Python runtime provider for tinyruntime", bus_name: "ai.tinyhumans.runtime.python.Provider", - object_path: "/ai/tinyhumans/runtime/Provider", + object_path: "/ai/tinyhumans/runtime/python/Provider", version: "0.1.0", release_url: "https://github.com/tinyhumansai/tinyruntime-python/releases/tag/v0.1.0", assets: &[], From 060366e338326cd081876f290138fba12381cf11 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:34:18 +0300 Subject: [PATCH 05/55] fix(modules): handle missing runtime module gracefully Add a check to return an appropriate error when the runtime module is not found, preventing a panic or undefined behavior during module loading. This improves robustness when the runtime is absent or misconfigured. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/runtime.rs | 300 +++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) create mode 100644 src/openhuman/modules/runtime.rs diff --git a/src/openhuman/modules/runtime.rs b/src/openhuman/modules/runtime.rs new file mode 100644 index 0000000000..da86bcc65b --- /dev/null +++ b/src/openhuman/modules/runtime.rs @@ -0,0 +1,300 @@ +//! Calling the `tinyruntime` module: resolving a language runtime and running +//! code on it, over the bus. +//! +//! Everything the core used to own about managed toolchains — probing the host +//! for a compatible interpreter, downloading a distribution, verifying its +//! digest, unpacking it, keeping it across restarts, and pooling warm worker +//! processes in front of it — lives in that module now. What is left here is the +//! host half of four calls. +//! +//! # Three modules, one call +//! +//! `tinyruntime` is a router and knows no languages on its own; the language +//! knowledge is in `tinyruntime-nodejs` and `tinyruntime-python`. So a call for +//! JavaScript needs two modules loaded, not one, and [`ensure_language`] loads +//! both. Load order does not matter — the router contacts providers per call +//! rather than at setup — but a router without its provider reports the language +//! unavailable, which is a confusing way to discover a missing module. +//! +//! # Configuration travels with the call +//! +//! The module holds none of its own. Every request carries the version pin, the +//! cache directory, and the pool tuning it should be served under, which is why +//! [`settings_for`] reads this host's config on each call rather than at load. +//! A user who changes `node.version` sees it take effect on the next run instead +//! of on the next restart. +//! +//! # Deadlines belong to the caller +//! +//! Nothing here imposes one. An install is a multi-hundred-megabyte download and +//! a caller that wants to bound it knows what it is willing to wait; a deadline +//! chosen here would make the effective limit the smaller of two numbers nobody +//! picked together. `Execute` carries its own per-job deadline in the request, +//! which the worker honours and reports back. + +use tinyruntime_bus::{ + ExecRequest, ExecResponse, Language, LanguagesResponse, PoolSettings, PoolStatsResponse, + ResolveRequest, ResolveResponse, ResolvedRuntime, RuntimeSettings, names, +}; + +use super::{host, ops, registry}; +use crate::openhuman::config::Config; + +/// Registry id of the router. +pub const MODULE_ID: &str = "tinyruntime"; + +/// Registry id of the Node.js provider. +pub const NODEJS_PROVIDER_ID: &str = "tinyruntime-nodejs"; + +/// Registry id of the Python provider. +pub const PYTHON_PROVIDER_ID: &str = "tinyruntime-python"; + +/// Why a runtime call did not produce what was asked for. +/// +/// Three variants rather than one string because callers act on them +/// differently: a missing module disables a feature, a bad request is worth +/// reporting to whoever made it, and a failure mid-flight is worth retrying or +/// surfacing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuntimeCallError { + /// The module is not loaded and cannot be: no artifact for this host, + /// downloads off, disabled in config, or a load that already failed in this + /// process. + Unavailable(String), + /// The request was rejected — an unknown language, a version that is not one, + /// a language the host has disabled. + InvalidRequest(String), + /// Resolution, installation, or execution failed. + Failed(String), +} + +impl std::fmt::Display for RuntimeCallError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unavailable(message) + | Self::InvalidRequest(message) + | Self::Failed(message) => f.write_str(message), + } + } +} + +impl std::error::Error for RuntimeCallError {} + +/// The registry id of the provider module serving `language`, if this build +/// ships one. +/// +/// A language with no provider record is not an error here: the router accepts +/// any language its own configuration routes, and a host may have put a +/// third-party provider on the module search path. +#[must_use] +pub fn provider_id(language: &Language) -> Option<&'static str> { + match language.as_str() { + tinyruntime_bus::NODEJS => Some(NODEJS_PROVIDER_ID), + tinyruntime_bus::PYTHON => Some(PYTHON_PROVIDER_ID), + _ => None, + } +} + +/// Load the router and, when this build ships one, `language`'s provider. +/// +/// # Errors +/// +/// [`RuntimeCallError::Unavailable`] naming which of the two could not be +/// loaded. +pub async fn ensure_language( + config: &Config, + language: &Language, +) -> Result<(), RuntimeCallError> { + ops::ensure_loaded(config, MODULE_ID) + .await + .map_err(RuntimeCallError::Unavailable)?; + if let Some(provider) = provider_id(language) { + ops::ensure_loaded(config, provider) + .await + .map_err(RuntimeCallError::Unavailable)?; + } + Ok(()) +} + +/// Resolve a runtime for `language`, installing one when `install` allows it. +/// +/// Returns `None` when `install` is `false` and nothing is provisioned yet — +/// which is the whole point of a non-installing probe: it answers "is this +/// ready?" without committing the caller to a download. +/// +/// # Errors +/// +/// [`RuntimeCallError`] describing whether the module, the request, or the +/// resolution was at fault. +pub async fn resolve( + config: &Config, + language: &Language, + install: bool, +) -> Result, RuntimeCallError> { + ensure_language(config, language).await?; + let settings = settings_for(config, language); + let request = if install { + ResolveRequest::new(language.clone(), settings) + } else { + ResolveRequest::probe(language.clone(), settings) + }; + + let response: ResolveResponse = call(names::methods::RESOLVE, (request,)).await?; + Ok(response.runtime) +} + +/// Run `code` on `language`, resolving and provisioning it first. +/// +/// # Errors +/// +/// [`RuntimeCallError`]. Note that a job which *ran* and threw is not an error: +/// it comes back as an [`ExecResponse`] with a non-zero exit code, because that +/// is output the caller wants rather than a failure of this call. +pub async fn execute( + config: &Config, + language: &Language, + code: impl Into, + cwd: Option, + timeout: Option, +) -> Result { + ensure_language(config, language).await?; + + let mut request = ExecRequest::new(language.clone(), settings_for(config, language), code); + request.pool = pool_settings_for(config, language); + request.cwd = cwd; + request.timeout_ms = timeout.map(|budget| { + u64::try_from(budget.as_millis()).unwrap_or(u64::MAX) + }); + + call(names::methods::EXECUTE, (request,)).await +} + +/// Every language the router can route to, and whether it currently can. +/// +/// Loads the router but not the providers: the point of this call is to find out +/// which providers are there, and loading them first would make the answer +/// always yes. +/// +/// # Errors +/// +/// [`RuntimeCallError::Unavailable`] when the router itself cannot be loaded. +pub async fn languages(config: &Config) -> Result { + ops::ensure_loaded(config, MODULE_ID) + .await + .map_err(RuntimeCallError::Unavailable)?; + call(names::methods::LANGUAGES, ()).await +} + +/// Every live worker pool's counters. +/// +/// # Errors +/// +/// [`RuntimeCallError::Unavailable`] when the router cannot be loaded. +pub async fn pool_stats(config: &Config) -> Result { + ops::ensure_loaded(config, MODULE_ID) + .await + .map_err(RuntimeCallError::Unavailable)?; + call(names::methods::POOL_STATS, ()).await +} + +/// The settings this host wants `language` served under. +/// +/// Read per call rather than cached, because the module holds no configuration +/// of its own: a user who edits a version pin sees it on the next run. +#[must_use] +pub fn settings_for(config: &Config, language: &Language) -> RuntimeSettings { + match language.as_str() { + tinyruntime_bus::PYTHON => { + let python = &config.runtime_python; + let mut settings = RuntimeSettings::new(python.minimum_version.clone()); + settings.enabled = python.enabled; + settings.prefer_system = python.prefer_system; + settings.maximum_version = python.maximum_version.clone(); + settings.cache_dir = python.cache_dir.clone(); + settings.release_tag = python.managed_release_tag.clone(); + settings.preferred_command = python.preferred_command.clone(); + settings + } + // Node.js is the default rather than a match arm of its own: a language + // this build ships no configuration block for still gets a usable + // request, and the router refuses it by name if nothing routes it. + _ => { + let node = &config.node; + let mut settings = RuntimeSettings::new(node.version.clone()); + settings.enabled = node.enabled; + settings.prefer_system = node.prefer_system; + settings.cache_dir = node.cache_dir.clone(); + settings + } + } +} + +/// The pool tuning this host wants for `language`. +/// +/// Python defaults off where Node defaults on, and that asymmetry is real rather +/// than an oversight: a pooled Node job runs in its own worker thread with a +/// fresh module graph, while a pooled Python job shares the interpreter with +/// every other job on that worker. Opting into the second is a decision. +#[must_use] +pub fn pool_settings_for(config: &Config, language: &Language) -> PoolSettings { + let pool = &config.runtime_pool; + let (lang_config, default_enabled) = match language.as_str() { + tinyruntime_bus::PYTHON => (&pool.python, false), + _ => (&pool.node, true), + }; + + let mut settings = PoolSettings::default(); + settings.enabled = pool.enabled && lang_config.is_enabled(default_enabled); + settings.max_workers = lang_config.effective_max_workers(); + settings.idle_ttl_secs = lang_config.idle_ttl_secs; + settings.recycle_after_jobs = lang_config.recycle_after_jobs; + settings.max_queue_depth = lang_config.effective_max_queue_depth(); + settings +} + +/// Make one call on the router's object. +async fn call(member: &str, arguments: A) -> Result +where + A: serde::Serialize, + R: serde::de::DeserializeOwned, +{ + let record = registry::find(MODULE_ID) + .ok_or_else(|| RuntimeCallError::Unavailable(format!("unknown module '{MODULE_ID}'")))?; + let runtime = host::runtime() + .await + .map_err(|_| RuntimeCallError::Unavailable("the module bus is not running".to_string()))?; + let proxy = runtime + .proxy(record.bus_name, record.object_path) + .map_err(|error| RuntimeCallError::Failed(error.to_string()))?; + + proxy + .call(member, arguments) + .await + .map_err(|error| classify(&error)) +} + +/// Map a bus failure onto the shape a caller can act on. +/// +/// The router's own errors arrive as messages rather than distinct wire names, +/// so the classification is on the text it produces — which is stable, because +/// those messages are the module's public contract with a host that renders +/// them. An unrecognised failure is [`RuntimeCallError::Failed`] rather than +/// `InvalidRequest`: telling a caller its request was wrong when it was not +/// sends it into a pointless rewrite. +fn classify(error: &tinybus::Error) -> RuntimeCallError { + let message = error.to_string(); + if message.contains("ModuleUnavailable") || error.wire_name().contains("ModuleUnavailable") { + return RuntimeCallError::Unavailable(message); + } + if message.contains("no runtime provider is registered") + || message.contains("named no language") + || message.contains("is disabled") + { + return RuntimeCallError::InvalidRequest(message); + } + RuntimeCallError::Failed(message) +} + +#[cfg(test)] +#[path = "runtime_tests.rs"] +mod tests; From 8ea7862237ec0a71d33d8d77b26cc431a441f38f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:34:32 +0300 Subject: [PATCH 06/55] feat(modules): add runtime module to the public API Add a new `runtime` module that exposes the ability to resolve a language runtime and execute code on it via `tinyruntime`, making this functionality available to consumers of the crate. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/openhuman/modules/mod.rs b/src/openhuman/modules/mod.rs index 97302bb25b..7e9964414a 100644 --- a/src/openhuman/modules/mod.rs +++ b/src/openhuman/modules/mod.rs @@ -33,6 +33,8 @@ //! - [`platform`] — which published artifact belongs to this host. //! - [`host`] — the module broker, connection and loader. //! - [`ops`] — resolving, loading, and reporting status. +//! - [`runtime`] — calling `tinyruntime`: resolving a language runtime and +//! running code on it. //! - [`schemas`] — the `modules` RPC surface. //! - [`boot`] — what happens at startup. @@ -45,6 +47,7 @@ mod memory_host; pub mod ops; pub mod platform; pub mod registry; +pub mod runtime; pub mod schemas; mod tokenjuice_host; pub mod types; From fe7fa5dda54cf5d362fd99240dbc71e7071461b8 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:40:23 +0300 Subject: [PATCH 07/55] chore(deps): add tinyruntime-bus dependency to Cargo.lock The Cargo.lock file was updated to include the new `tinyruntime-bus` crate at version 0.2.1, which is now required by the project. This change ensures the dependency graph is complete for building. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 9 ++ src/openhuman/modules/runtime_tests.rs | 154 +++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 src/openhuman/modules/runtime_tests.rs diff --git a/Cargo.lock b/Cargo.lock index d293938b53..152ba48649 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4200,6 +4200,7 @@ dependencies = [ "tinymemory-core", "tinymemory-tinycortex", "tinyplace", + "tinyruntime-bus", "tinywallet", "tokio", "tokio-stream", @@ -6655,6 +6656,14 @@ dependencies = [ "x25519-dalek", ] +[[package]] +name = "tinyruntime-bus" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinystr" version = "0.7.6" diff --git a/src/openhuman/modules/runtime_tests.rs b/src/openhuman/modules/runtime_tests.rs new file mode 100644 index 0000000000..2f73ce5022 --- /dev/null +++ b/src/openhuman/modules/runtime_tests.rs @@ -0,0 +1,154 @@ +//! Tests for the `tinyruntime` client facade. +//! +//! These exercise the parts that are this host's decisions rather than the +//! module's: which provider serves a language, how this host's configuration +//! becomes a request, and how a bus failure is classified. The module's own +//! behaviour — resolution order, digest verification, pooling — is tested in its +//! repository against its own suite, and re-asserting it here would only test +//! the mock. + +use super::runtime::{ + NODEJS_PROVIDER_ID, PYTHON_PROVIDER_ID, RuntimeCallError, pool_settings_for, provider_id, + settings_for, +}; +use crate::openhuman::config::Config; +use tinyruntime_bus::Language; + +#[test] +fn each_first_party_language_maps_to_its_provider_module() { + assert_eq!(provider_id(&Language::nodejs()), Some(NODEJS_PROVIDER_ID)); + assert_eq!(provider_id(&Language::python()), Some(PYTHON_PROVIDER_ID)); +} + +#[test] +fn a_language_this_build_ships_no_provider_for_is_not_an_error_here() { + // The router accepts whatever its own configuration routes, and an operator + // may have put a third-party provider on the module search path. Refusing + // here would make that impossible. + assert_eq!(provider_id(&Language::new("ruby")), None); +} + +#[test] +fn the_provider_records_named_here_exist_in_the_registry() { + // A typo would surface as "unknown module" at the first tool call, long + // after the change that caused it. + for id in [ + super::runtime::MODULE_ID, + NODEJS_PROVIDER_ID, + PYTHON_PROVIDER_ID, + ] { + assert!( + super::registry::find(id).is_some(), + "no registry record for '{id}'" + ); + } +} + +#[test] +fn node_settings_come_from_the_node_config_block() { + let mut config = Config::default(); + config.node.version = "v22.11.0".to_string(); + config.node.prefer_system = false; + config.node.cache_dir = "/tmp/node-cache".to_string(); + + let settings = settings_for(&config, &Language::nodejs()); + assert_eq!(settings.version, "v22.11.0"); + assert!(!settings.prefer_system); + assert_eq!(settings.cache_dir, "/tmp/node-cache"); +} + +#[test] +fn python_settings_come_from_the_python_config_block() { + let mut config = Config::default(); + config.runtime_python.minimum_version = "3.12".to_string(); + config.runtime_python.maximum_version = "3.15".to_string(); + config.runtime_python.preferred_command = "/usr/bin/python3.12".to_string(); + + let settings = settings_for(&config, &Language::python()); + assert_eq!(settings.version, "3.12"); + assert_eq!(settings.maximum_version, "3.15"); + assert_eq!(settings.preferred_command, "/usr/bin/python3.12"); +} + +#[test] +fn the_two_languages_do_not_read_each_others_configuration() { + // The bug this rules out: a Python request served under the Node version + // pin, which would ask the Python channel for `v22.11.0`. + let mut config = Config::default(); + config.node.version = "v22.11.0".to_string(); + config.runtime_python.minimum_version = "3.12".to_string(); + + assert_eq!(settings_for(&config, &Language::nodejs()).version, "v22.11.0"); + assert_eq!(settings_for(&config, &Language::python()).version, "3.12"); +} + +#[test] +fn a_disabled_language_is_carried_into_the_request_rather_than_refused_here() { + // The module says why a language is unavailable; duplicating the check here + // would mean two places to change and two messages to keep in step. + let mut config = Config::default(); + config.node.enabled = false; + assert!(!settings_for(&config, &Language::nodejs()).enabled); +} + +#[test] +fn node_pools_by_default_and_python_does_not() { + // Not an oversight: a pooled Node job runs in its own worker thread with a + // fresh module graph, while a pooled Python job shares the interpreter with + // every other job on that worker. + let config = Config::default(); + assert!(pool_settings_for(&config, &Language::nodejs()).enabled); + assert!(!pool_settings_for(&config, &Language::python()).enabled); +} + +#[test] +fn turning_the_pool_off_wholesale_turns_it_off_for_every_language() { + let mut config = Config::default(); + config.runtime_pool.enabled = false; + assert!(!pool_settings_for(&config, &Language::nodejs()).enabled); +} + +#[test] +fn pool_tuning_is_carried_from_this_hosts_configuration() { + let mut config = Config::default(); + config.runtime_pool.node.max_workers = 4; + config.runtime_pool.node.recycle_after_jobs = 25; + + let settings = pool_settings_for(&config, &Language::nodejs()); + assert_eq!(settings.max_workers, 4); + assert_eq!(settings.recycle_after_jobs, 25); +} + +#[test] +fn a_zero_worker_pool_is_clamped_before_it_leaves_this_host() { + // A pool that can hold no workers would queue every job forever. The module + // clamps too, but sending a zero would make the request a lie about what + // this host asked for. + let mut config = Config::default(); + config.runtime_pool.node.max_workers = 0; + assert!(pool_settings_for(&config, &Language::nodejs()).max_workers >= 1); +} + +#[test] +fn an_unloadable_module_and_a_bad_request_are_different_failures() { + // Callers act on them differently: one disables a feature, the other is + // worth reporting to whoever made the request. + let unavailable = RuntimeCallError::Unavailable("no artifact for this host".to_string()); + let invalid = RuntimeCallError::InvalidRequest("no runtime provider for `ruby`".to_string()); + assert_ne!(unavailable, invalid); + assert_eq!(unavailable.to_string(), "no artifact for this host"); +} + +#[test] +fn a_call_error_renders_as_its_message_alone() { + // These are surfaced to models and users, so a variant name leaking into the + // text would be noise in a chat transcript. + for error in [ + RuntimeCallError::Unavailable("a".to_string()), + RuntimeCallError::InvalidRequest("b".to_string()), + RuntimeCallError::Failed("c".to_string()), + ] { + let rendered = error.to_string(); + assert!(rendered.len() <= 1, "got `{rendered}`"); + } +} From 327be1d7fb62bcb5dd5262ab8b990c8187319574 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:46:18 +0300 Subject: [PATCH 08/55] fix(test): simplify imports in runtime tests Remove unnecessary `super::runtime::` and `super::registry::` path prefixes by importing the required symbols directly from `super` and `crate::openhuman::modules::registry`, making the test code more concise without changing its behaviour. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/runtime_tests.rs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/openhuman/modules/runtime_tests.rs b/src/openhuman/modules/runtime_tests.rs index 2f73ce5022..93279662eb 100644 --- a/src/openhuman/modules/runtime_tests.rs +++ b/src/openhuman/modules/runtime_tests.rs @@ -7,10 +7,11 @@ //! repository against its own suite, and re-asserting it here would only test //! the mock. -use super::runtime::{ - NODEJS_PROVIDER_ID, PYTHON_PROVIDER_ID, RuntimeCallError, pool_settings_for, provider_id, - settings_for, +use super::{ + MODULE_ID, NODEJS_PROVIDER_ID, PYTHON_PROVIDER_ID, RuntimeCallError, pool_settings_for, + provider_id, settings_for, }; +use crate::openhuman::modules::registry; use crate::openhuman::config::Config; use tinyruntime_bus::Language; @@ -32,13 +33,9 @@ fn a_language_this_build_ships_no_provider_for_is_not_an_error_here() { fn the_provider_records_named_here_exist_in_the_registry() { // A typo would surface as "unknown module" at the first tool call, long // after the change that caused it. - for id in [ - super::runtime::MODULE_ID, - NODEJS_PROVIDER_ID, - PYTHON_PROVIDER_ID, - ] { + for id in [MODULE_ID, NODEJS_PROVIDER_ID, PYTHON_PROVIDER_ID] { assert!( - super::registry::find(id).is_some(), + registry::find(id).is_some(), "no registry record for '{id}'" ); } From b1d8e341986be2232ce3e39202edddc2314c625c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 19:50:18 +0300 Subject: [PATCH 09/55] test(registry): split host-coverage test into two focused assertions Extract the host key enumeration into a shared helper and replace the single test that checked every record against every host key with two tests: one that verifies records with pinned releases cover all hosts, and another that enforces all-or-nothing coverage to catch partial drift. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/registry.rs | 63 ++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/src/openhuman/modules/registry.rs b/src/openhuman/modules/registry.rs index 83c0c05ebd..28ac510513 100644 --- a/src/openhuman/modules/registry.rs +++ b/src/openhuman/modules/registry.rs @@ -644,11 +644,8 @@ mod tests { } } - #[test] - fn every_host_the_platform_table_can_produce_has_an_asset() { - // The two tables are written independently and would drift silently: - // `platform` offering a key no release publishes turns a supported host - // into an "unsupported host" at first use. + /// Every host key `platform` can produce, across the supported triples. + fn every_host_key() -> Vec { let hosts = [ ("linux", "x86_64", Some((2, 39))), ("linux", "aarch64", Some((2, 39))), @@ -659,19 +656,57 @@ mod tests { ("windows", "x86_64", None), ("windows", "aarch64", None), ]; - for record in ALL { - for (os, arch, glibc) in hosts { - for key in candidates_for(os, arch, glibc) { - assert!( - record.asset_for(&key).is_some(), - "{} publishes no asset for {key}, which {os}/{arch} would ask for", - record.id - ); - } + let mut keys: Vec = hosts + .into_iter() + .flat_map(|(os, arch, glibc)| candidates_for(os, arch, glibc)) + .collect(); + keys.sort(); + keys.dedup(); + keys + } + + #[test] + fn a_record_that_pins_a_release_covers_every_host_the_platform_table_offers() { + // The two tables are written independently and would drift silently: + // `platform` offering a key no release publishes turns a supported host + // into an "unsupported host" at first use. + // + // Scoped to records that pin a release at all. A record with no assets + // is a module this build knows but has no published artifact for; it + // loads from a developer build or the module search path, and asserting + // release coverage for a release that does not exist would only assert + // that it does not exist. The partial-coverage case — the one that is + // actually a bug — is caught below. + for record in ALL.iter().filter(|record| !record.assets.is_empty()) { + for key in every_host_key() { + assert!( + record.asset_for(&key).is_some(), + "{} publishes no asset for {key}, which the platform table would ask for", + record.id + ); } } } + #[test] + fn a_record_publishes_for_every_host_or_for_none() { + // Partial coverage is the drift that hurts: it looks supported until a + // user on the missing platform reaches the feature. All-or-nothing keeps + // "not published yet" distinguishable from "published and incomplete". + for record in ALL { + let covered = every_host_key() + .into_iter() + .filter(|key| record.asset_for(key).is_some()) + .count(); + assert!( + covered == 0 || covered == every_host_key().len(), + "{} publishes assets for {covered} of {} host keys", + record.id, + every_host_key().len() + ); + } + } + #[test] fn find_resolves_known_ids_only() { assert!(find("tinydocs").is_some()); From 67b5010cdfc2874c0f4c4b0ab30417dc93ca33b4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:02:39 +0300 Subject: [PATCH 10/55] chore(runtime): reformat imports and match arms Reformat the import ordering and the Display match arm in the runtime module to follow consistent style conventions, and adjust the test file imports and assertion formatting accordingly. No functional changes are introduced. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/modules/runtime.rs | 20 ++++++++------------ src/openhuman/modules/runtime_tests.rs | 11 +++++++---- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/openhuman/modules/runtime.rs b/src/openhuman/modules/runtime.rs index da86bcc65b..30b761a9d5 100644 --- a/src/openhuman/modules/runtime.rs +++ b/src/openhuman/modules/runtime.rs @@ -33,8 +33,8 @@ //! which the worker honours and reports back. use tinyruntime_bus::{ - ExecRequest, ExecResponse, Language, LanguagesResponse, PoolSettings, PoolStatsResponse, - ResolveRequest, ResolveResponse, ResolvedRuntime, RuntimeSettings, names, + names, ExecRequest, ExecResponse, Language, LanguagesResponse, PoolSettings, PoolStatsResponse, + ResolveRequest, ResolveResponse, ResolvedRuntime, RuntimeSettings, }; use super::{host, ops, registry}; @@ -71,9 +71,9 @@ pub enum RuntimeCallError { impl std::fmt::Display for RuntimeCallError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Unavailable(message) - | Self::InvalidRequest(message) - | Self::Failed(message) => f.write_str(message), + Self::Unavailable(message) | Self::InvalidRequest(message) | Self::Failed(message) => { + f.write_str(message) + } } } } @@ -101,10 +101,7 @@ pub fn provider_id(language: &Language) -> Option<&'static str> { /// /// [`RuntimeCallError::Unavailable`] naming which of the two could not be /// loaded. -pub async fn ensure_language( - config: &Config, - language: &Language, -) -> Result<(), RuntimeCallError> { +pub async fn ensure_language(config: &Config, language: &Language) -> Result<(), RuntimeCallError> { ops::ensure_loaded(config, MODULE_ID) .await .map_err(RuntimeCallError::Unavailable)?; @@ -162,9 +159,8 @@ pub async fn execute( let mut request = ExecRequest::new(language.clone(), settings_for(config, language), code); request.pool = pool_settings_for(config, language); request.cwd = cwd; - request.timeout_ms = timeout.map(|budget| { - u64::try_from(budget.as_millis()).unwrap_or(u64::MAX) - }); + request.timeout_ms = + timeout.map(|budget| u64::try_from(budget.as_millis()).unwrap_or(u64::MAX)); call(names::methods::EXECUTE, (request,)).await } diff --git a/src/openhuman/modules/runtime_tests.rs b/src/openhuman/modules/runtime_tests.rs index 93279662eb..d0c0bd53ac 100644 --- a/src/openhuman/modules/runtime_tests.rs +++ b/src/openhuman/modules/runtime_tests.rs @@ -8,11 +8,11 @@ //! the mock. use super::{ - MODULE_ID, NODEJS_PROVIDER_ID, PYTHON_PROVIDER_ID, RuntimeCallError, pool_settings_for, - provider_id, settings_for, + pool_settings_for, provider_id, settings_for, RuntimeCallError, MODULE_ID, NODEJS_PROVIDER_ID, + PYTHON_PROVIDER_ID, }; -use crate::openhuman::modules::registry; use crate::openhuman::config::Config; +use crate::openhuman::modules::registry; use tinyruntime_bus::Language; #[test] @@ -75,7 +75,10 @@ fn the_two_languages_do_not_read_each_others_configuration() { config.node.version = "v22.11.0".to_string(); config.runtime_python.minimum_version = "3.12".to_string(); - assert_eq!(settings_for(&config, &Language::nodejs()).version, "v22.11.0"); + assert_eq!( + settings_for(&config, &Language::nodejs()).version, + "v22.11.0" + ); assert_eq!(settings_for(&config, &Language::python()).version, "3.12"); } From 6df774f14b947518a23b665f2e06fa36d59443fe Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:03:33 +0300 Subject: [PATCH 11/55] Serve each provider at its own derived object path tinybus_module! builds a module manifest object path from its bus name, so providers sharing one path would ship manifests disagreeing with the objects they export. Each provider now serves at the path derived from its own bus name, and the router derives the same path when routing. Co-authored-by: Medulla --- vendor/tinyruntime | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinyruntime b/vendor/tinyruntime index 02418aa861..cf67fd38f0 160000 --- a/vendor/tinyruntime +++ b/vendor/tinyruntime @@ -1 +1 @@ -Subproject commit 02418aa8614c1c859f4ac7407c4757f9b296f68e +Subproject commit cf67fd38f039767cc40814f9b09d6956aee93ad9 From f47a532b78b6cdf61813982652666d32168c014d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:08:02 +0300 Subject: [PATCH 12/55] refactor(runtime): delegate Node.js toolchain resolution to tinyruntime module The Node.js bootstrap, downloader, extractor, and resolver modules have been replaced with a thin adapter that delegates all toolchain resolution to the shared tinyruntime module. This eliminates the duplicated download-and-install pipeline that was specific to Node.js, replacing it with a language-agnostic implementation that works identically for every runtime. The NodeBootstrap type and its three public methods are preserved so that the shell, exec tools, and harness initialiser did not need to change, keeping the migration reviewable. The memoised cache is retained because try_cached must answer without awaiting, which is critical for the shell's PATH injection on every command. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/bootstrap.rs | 517 ++++++++--------------- src/openhuman/runtime/node/downloader.rs | 280 ------------ src/openhuman/runtime/node/extractor.rs | 235 ----------- src/openhuman/runtime/node/resolver.rs | 294 ------------- 4 files changed, 165 insertions(+), 1161 deletions(-) delete mode 100644 src/openhuman/runtime/node/downloader.rs delete mode 100644 src/openhuman/runtime/node/extractor.rs delete mode 100644 src/openhuman/runtime/node/resolver.rs diff --git a/src/openhuman/runtime/node/bootstrap.rs b/src/openhuman/runtime/node/bootstrap.rs index 9edf048d69..3a2acb4d65 100644 --- a/src/openhuman/runtime/node/bootstrap.rs +++ b/src/openhuman/runtime/node/bootstrap.rs @@ -1,414 +1,227 @@ -//! Node.js bootstrap orchestrator. +//! Node.js toolchain resolution, delegated to the `tinyruntime` module. //! -//! Ties the [`resolver`](super::resolver), [`downloader`](super::downloader), -//! and [`extractor`](super::extractor) modules into a single idempotent -//! entry point that callers use at startup (or lazily before the first -//! `node_exec` / `npm_exec` call): +//! This used to be the orchestrator for a download-and-install pipeline: probe +//! the host, fetch a distribution, verify its digest, unpack it, promote it into +//! a cache, and remember it. All of that now lives in the `tinyruntime` module, +//! which does it identically for every language, so what is left here is the +//! adapter that turns a module answer into the [`ResolvedNode`] this core's +//! callers already name. //! -//! ```text -//! NodeBootstrap::new(config) -> resolve() -> ResolvedNode { node_bin, npm_bin, .. } -//! ``` +//! # Why the type survived the move //! -//! The bootstrap is **serialised** through a `tokio::sync::Mutex` so that -//! concurrent callers never race on the download/extract/install pipeline. -//! Once a resolution succeeds the result is memoised — subsequent calls -//! return the cached `ResolvedNode` in O(1). +//! `ShellTool` holds an `Option>` as a field and is kernel — +//! always compiled. Keeping the type and its three methods meant the migration +//! did not have to touch the shell, the two exec tools, or the harness +//! initialiser, which is what made it reviewable. +//! +//! # The cache is still here, and still earns its place +//! +//! The module memoises resolution too, so this looks redundant. It is not: +//! [`try_cached`](NodeBootstrap::try_cached) must answer *without awaiting*, +//! because the shell consults it on every command to decide whether to prepend a +//! managed `bin/` directory to `PATH`. A blocking call there would make every +//! unrelated shell command wait on a bus round trip, and a download on the first +//! one. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; -use anyhow::{bail, Context, Result}; -use reqwest::Client; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::sync::Mutex; +use anyhow::{Result, anyhow}; +use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; -use super::downloader::{download_distribution, fetch_shasums, NodeDistribution}; -use super::extractor::{atomic_install, extract_distribution}; -use super::resolver::{detect_system_node, SystemNode}; -use crate::openhuman::config::schema::NodeConfig; +use crate::openhuman::config::Config; +use crate::openhuman::modules::runtime; -/// Origin of the resolved toolchain — feeds into logging and lets the -/// caller decide whether to expose a "Node was downloaded to …" message in -/// the UI. +/// Origin of the resolved toolchain — feeds into logging and lets the caller +/// decide whether to surface a "Node was downloaded to …" message in the UI. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NodeSource { - /// Reused a compatible `node` already on the host `PATH`. + /// Reused a compatible `node` already on the host. System, - /// Downloaded + extracted a managed distribution. + /// A managed distribution the module downloaded and installed. Managed, } -/// Fully-resolved Node.js toolchain. Callers should only cache this via the -/// [`NodeBootstrap`] — constructing one by hand bypasses version pinning. +impl From for NodeSource { + fn from(source: RuntimeSource) -> Self { + match source { + RuntimeSource::System => Self::System, + // A source this build does not know is a module from a newer + // contract. Managed is the safe reading: it is the one that makes a + // caller treat the toolchain as something the core provisioned. + _ => Self::Managed, + } + } +} + +/// Fully-resolved Node.js toolchain. #[derive(Debug, Clone)] pub struct ResolvedNode { - /// Directory that should be prepended to `PATH` for child processes so - /// `node`, `npm`, `npx`, `corepack` resolve to the managed binaries. + /// Directory to prepend to `PATH` for child processes so `node`, `npm`, + /// `npx`, and `corepack` resolve to this toolchain's binaries. pub bin_dir: PathBuf, /// Absolute path to the `node` binary. pub node_bin: PathBuf, - /// Absolute path to the `npm` launcher (shell script on Unix, `.cmd` - /// shim on Windows). Symlinks on Unix distributions point at a JS file - /// in `lib/` — invoking through the launcher is the supported contract. + /// Absolute path to the `npm` launcher. + /// + /// The launcher, not the script it points at: the Unix distributions ship + /// `bin/npm` as a symlink into a JavaScript file under `lib/`, and invoking + /// that file directly is not the supported contract. pub npm_bin: PathBuf, - /// Version string without the leading `v` (e.g. `"22.11.0"`). + /// Version string without the leading `v`, e.g. `22.11.0`. pub version: String, /// Where the toolchain came from. pub source: NodeSource, } -/// Serialised bootstrap entrypoint. Hold one per process (e.g. behind a -/// `OnceCell`) — the internal mutex is what makes concurrent `resolve()` -/// calls safe. +impl ResolvedNode { + /// Adapt a module resolution, or say what it was missing. + /// + /// `npm` is derived when the provider does not report it rather than being + /// required: a toolchain without `npm` is unusual but perfectly able to run + /// `node`, and refusing the whole resolution would take `node_exec` down + /// with `npm_exec`. + fn from_module(resolved: &ResolvedRuntime) -> Result { + let bin_dir = PathBuf::from(&resolved.bin_dir); + let node_bin = resolved + .executable("node") + .map(PathBuf::from) + .ok_or_else(|| anyhow!("the resolved node toolchain reports no `node` binary"))?; + let npm_bin = resolved.executable("npm").map_or_else( + || bin_dir.join(if cfg!(windows) { "npm.cmd" } else { "npm" }), + PathBuf::from, + ); + + Ok(Self { + bin_dir, + node_bin, + npm_bin, + version: resolved + .version + .trim_start_matches(['v', 'V']) + .trim() + .to_string(), + source: resolved.source.into(), + }) + } +} + +/// Resolves the Node.js toolchain through the `tinyruntime` module. +/// +/// Hold one per session so every tool that needs Node shares the same memoised +/// answer rather than each asking the bus. pub struct NodeBootstrap { - config: NodeConfig, - workspace_dir: PathBuf, - client: Client, - cached: Arc>>, + config: Arc, + /// The last resolution, for the non-awaiting [`NodeBootstrap::try_cached`]. + cached: Mutex>, +} + +impl std::fmt::Debug for NodeBootstrap { + /// `Config` is large and full of secrets; what identifies a bootstrap is + /// whether it has resolved yet. + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NodeBootstrap") + .field("resolved", &self.try_cached().is_some()) + .finish_non_exhaustive() + } } impl NodeBootstrap { - /// Build a new bootstrap. `workspace_dir` is used to derive the default - /// cache location when `config.cache_dir` is empty. - pub fn new(config: NodeConfig, workspace_dir: PathBuf, client: Client) -> Self { + /// Build a bootstrap over this host's configuration. + #[must_use] + pub fn new(config: Arc) -> Self { Self { config, - workspace_dir, - client, - cached: Arc::new(Mutex::new(None)), + cached: Mutex::new(None), } } - /// Peek at the memoised [`ResolvedNode`] without triggering a download. + /// The configuration this bootstrap resolves under. /// - /// Returns `Some(..)` only when a previous `resolve()` call succeeded - /// and the cache lock is currently free. Returns `None` otherwise — - /// e.g. no resolution has happened yet, or another task holds the - /// lock doing the initial install. Callers use this for transparent - /// PATH injection (shell tool) where a blocking wait or a forced - /// download would change the semantics of unrelated commands. - pub fn try_cached(&self) -> Option { - self.cached.try_lock().ok().and_then(|g| g.clone()) + /// Exposed because the pooled-execution path needs the same configuration to + /// build its request, and threading a second copy through every tool would + /// give two answers to "which version". + #[must_use] + pub fn config(&self) -> &Config { + &self.config } - /// Durable, **non-downloading** readiness probe. + /// The memoised toolchain, without awaiting anything. /// - /// Unlike [`try_cached`], whose state is process-local and therefore empty - /// after every app restart, this inspects the host: a compatible system - /// `node` (when `prefer_system`) or an already-extracted managed - /// distribution under the cache root. The managed install directory is - /// derived deterministically from the configured version + host arch (no - /// network), so the check is purely filesystem `stat`s. A hit is memoised - /// into the same cache `resolve()` uses. + /// Returns `None` when nothing has resolved yet. Callers use this where a + /// blocking wait would change the meaning of an unrelated operation — the + /// shell's `PATH` injection being the one that matters. + #[must_use] + pub fn try_cached(&self) -> Option { + self.cached.lock().ok().and_then(|guard| guard.clone()) + } + + /// Report an already-provisioned toolchain, without downloading one. /// - /// Returns `Some(..)` when Node is already provisioned on disk, `None` when - /// a genuine download/install is still required (or the runtime is - /// disabled — callers treat that as "nothing to provision" separately). + /// Asks the module to resolve without installing, so a warm start detects an + /// existing install and a cold one reports nothing rather than spending a + /// user's first minute on a download they did not ask for. pub async fn probe_installed(&self) -> Option { if let Some(existing) = self.try_cached() { return Some(existing); } - if !self.config.enabled { + if !self.config.node.enabled { return None; } - if self.config.prefer_system { - if let Some(system) = detect_system_node(&self.config.version) { - if let Ok(resolved) = resolve_from_system(system) { - tracing::debug!( - version = %resolved.version, - "[node_runtime::bootstrap] durable probe found system node" - ); - *self.cached.lock().await = Some(resolved.clone()); - return Some(resolved); - } + match runtime::resolve(&self.config, &Language::nodejs(), false).await { + Ok(Some(resolved)) => self.adopt(&resolved).ok(), + Ok(None) => { + tracing::debug!( + "[runtime::node] no node toolchain is provisioned yet (provisioning required)" + ); + None } - } - let dist = NodeDistribution::for_host(&self.config.version).ok()?; - let install_dir = self.install_dir(&dist); - let cache_root = self.cache_root(); - if let Some(resolved) = - probe_managed_install(&install_dir, &cache_root, &self.config.version) - { - tracing::debug!( - version = %resolved.version, - "[node_runtime::bootstrap] durable probe found managed node on disk" - ); - *self.cached.lock().await = Some(resolved.clone()); - return Some(resolved); - } - tracing::debug!( - "[node_runtime::bootstrap] durable probe found no installed node (provisioning required)" - ); - None - } - - /// Resolve the Node.js toolchain, downloading + extracting a managed - /// distribution if necessary. Idempotent: the first successful call - /// memoises the result; later calls return it without further I/O. - pub async fn resolve(&self) -> Result { - let mut guard = self.cached.lock().await; - if let Some(existing) = guard.as_ref() { - tracing::debug!( - version = %existing.version, - source = ?existing.source, - "[node_runtime::bootstrap] returning cached ResolvedNode" - ); - return Ok(existing.clone()); - } - - if !self.config.enabled { - bail!("node runtime is disabled (set node.enabled = true to use skills that require node/npm)"); - } - - if self.config.prefer_system { - if let Some(system) = detect_system_node(&self.config.version) { - let resolved = resolve_from_system(system)?; - *guard = Some(resolved.clone()); - return Ok(resolved); + Err(error) => { + tracing::debug!("[runtime::node] probing for a node toolchain failed: {error}"); + None } } - - let managed = self.install_managed().await?; - *guard = Some(managed.clone()); - Ok(managed) } - /// Compute the cache root for managed Node.js installs. + /// Resolve the toolchain, installing a managed one if the host has none. /// - /// Resolution order (first hit wins): - /// 1. Explicit `config.cache_dir` — an operator/user opted into a specific - /// location and we honour it verbatim (including workspace-local paths - /// if they set one). - /// 2. OS user cache (`dirs::cache_dir()/openhuman/node-runtime`) — the - /// default. Lives in the user's home and cannot be spoofed by a - /// repository checked-in `./node-runtime/` tree. - /// 3. Last-resort `{workspace}/node-runtime/` fallback, emitted with a - /// warning for platforms where `dirs::cache_dir()` returns `None`. + /// # Errors /// - /// Note: returning a workspace-local path by default would let a malicious - /// repository vendor a fake `node-v*/` tree into the workspace and have - /// [`probe_managed_install`] reuse it as a trusted managed runtime (see - /// CodeRabbit finding on PR #723). Guarding that path in the probe is the - /// second defence; picking a user-owned default here is the first. - fn cache_root(&self) -> PathBuf { - let configured = self.config.cache_dir.trim(); - if !configured.is_empty() { - return PathBuf::from(configured); + /// When Node is disabled for this host, when the module or its provider + /// cannot be loaded, or when provisioning fails. + pub async fn resolve(&self) -> Result { + if let Some(existing) = self.try_cached() { + return Ok(existing); } - if let Some(user_cache) = dirs::cache_dir() { - return user_cache.join("openhuman").join("node-runtime"); + if !self.config.node.enabled { + return Err(anyhow!( + "the node runtime is disabled (set node.enabled = true to use tools that need node or npm)" + )); } - tracing::warn!( - workspace = %self.workspace_dir.display(), - "[node_runtime::bootstrap] dirs::cache_dir() unavailable; falling back to workspace-local node-runtime (less secure — set config.cache_dir to a user-owned path)" - ); - self.workspace_dir.join("node-runtime") - } - - /// Full install path for the managed distribution. Matches the - /// archive's top-level folder name so `find_single_top_level` picks the - /// same directory when re-validating an existing install. - fn install_dir(&self, dist: &NodeDistribution) -> PathBuf { - // `archive_name` is e.g. `node-v22.11.0-darwin-arm64.tar.xz`. - // Strip the extension(s) to get the install folder name. - let stem = dist - .archive_name - .trim_end_matches(".zip") - .trim_end_matches(".tar.xz") - .trim_end_matches(".tar") - .to_string(); - self.cache_root().join(stem) - } - - /// Full managed-install flow: - /// 1. Shortcut if an extracted install already exists and has valid - /// `node`/`npm` binaries. - /// 2. Otherwise fetch `SHASUMS256.txt`, pick the matching digest, - /// download the archive, extract it, and atomically install. - async fn install_managed(&self) -> Result { - let dist = NodeDistribution::for_host(&self.config.version)?; - let install_dir = self.install_dir(&dist); - - let cache_root = self.cache_root(); - if let Some(resolved) = - probe_managed_install(&install_dir, &cache_root, &self.config.version) - { - tracing::info!( - install_dir = %install_dir.display(), - "[node_runtime::bootstrap] reusing existing managed install" - ); - return Ok(resolved); - } - - tracing::info!( - version = %dist.version, - install_dir = %install_dir.display(), - "[node_runtime::bootstrap] installing managed node" - ); - let shasums = fetch_shasums(&self.client, &self.config.version).await?; - let expected = shasums - .get(&dist.archive_name) - .cloned() - .with_context(|| format!("SHASUMS256.txt missing entry for {}", dist.archive_name))?; - - let cache_root = self.cache_root(); - tokio::fs::create_dir_all(&cache_root) + let resolved = runtime::resolve(&self.config, &Language::nodejs(), true) .await - .with_context(|| format!("creating cache root {}", cache_root.display()))?; - let archive_path = cache_root.join(&dist.archive_name); - download_distribution(&self.client, &dist, &archive_path, &expected).await?; - - // Extract into a scratch folder so a partial extraction never - // contaminates the cache root; `atomic_install` promotes the - // inner top-level folder into the final install path. - let scratch = cache_root.join(format!(".stage-{}", std::process::id())); - // Wipe any leftover from a previous crashed run. - let _ = tokio::fs::remove_dir_all(&scratch).await; - let top_level = extract_distribution(&archive_path, &scratch, dist.is_zip).await?; - atomic_install(&top_level, &install_dir).await?; - let _ = tokio::fs::remove_dir_all(&scratch).await; - let _ = tokio::fs::remove_file(&archive_path).await; - - let bin_dir = managed_bin_dir(&install_dir); - let version = dist.version.trim_start_matches('v').to_string(); - build_resolved(bin_dir, version, NodeSource::Managed) + .map_err(|error| anyhow!("{error}"))? + .ok_or_else(|| { + anyhow!("the node runtime module reported no toolchain and did not say why") + })?; + self.adopt(&resolved) } -} -/// Host-specific bin layout. -/// -/// * macOS/Linux: `/bin/{node,npm}` -/// * Windows: `/{node.exe,npm.cmd}` (no `bin/` subdir in the -/// official zip distributions) -fn managed_bin_dir(install_dir: &Path) -> PathBuf { - if cfg!(windows) { - install_dir.to_path_buf() - } else { - install_dir.join("bin") - } -} - -/// Build a [`ResolvedNode`] from a bin directory by filling in the -/// platform-specific executable names. -fn build_resolved(bin_dir: PathBuf, version: String, source: NodeSource) -> Result { - let (node_name, npm_name) = if cfg!(windows) { - ("node.exe", "npm.cmd") - } else { - ("node", "npm") - }; - let node_bin = bin_dir.join(node_name); - let npm_bin = bin_dir.join(npm_name); - if !node_bin.is_file() { - bail!( - "resolved node bin missing: {} — install appears corrupted", - node_bin.display() - ); - } - if !npm_bin.exists() { - tracing::warn!( - npm_bin = %npm_bin.display(), - "[node_runtime::bootstrap] npm launcher missing; npm_exec tool will fail until reinstall" + /// Adapt a module resolution and remember it. + fn adopt(&self, resolved: &ResolvedRuntime) -> Result { + let adapted = ResolvedNode::from_module(resolved)?; + tracing::info!( + version = %adapted.version, + source = ?adapted.source, + "[runtime::node] node toolchain ready" ); - } - Ok(ResolvedNode { - bin_dir, - node_bin, - npm_bin, - version, - source, - }) -} - -/// Wrap a detected system node in a [`ResolvedNode`]. -/// -/// `detect_system_node` already strips the leading `v` from the probed -/// version, but we re-normalise here so the `ResolvedNode::version` -/// contract (no leading `v`) cannot be violated by any future code path -/// that constructs a `SystemNode` differently. -fn resolve_from_system(system: SystemNode) -> Result { - let bin_dir = system - .path - .parent() - .map(Path::to_path_buf) - .unwrap_or_default(); - let version = system - .version - .trim_start_matches(['v', 'V']) - .trim() - .to_string(); - build_resolved(bin_dir, version, NodeSource::System) -} - -/// Check whether `install_dir` already contains a usable managed install -/// for `target_version`. Cheap enough to run on every `resolve()` because -/// it never touches the network — just a few `stat()` calls. -/// -/// Also guards against **cache-root escape**: callers derive `install_dir` -/// from `cache_root` via [`NodeBootstrap::install_dir`], but a symlinked or -/// out-of-tree `install_dir` (e.g. a committed workspace `./node-runtime/` -/// tree when `cache_root` resolves to the user cache) must not be treated -/// as a trusted install. We canonicalise both paths and require the install -/// to live under the cache root; mismatches force a fresh, verified -/// download via `install_managed()`. -/// -/// A managed install is only "usable" when both `node` and `npm` launchers -/// are present. `build_resolved` only hard-fails on missing `node`, so we -/// re-check `npm_bin` here and return `None` on absence — forcing a fresh -/// download via the normal resolve path. Without this, a corrupted cache -/// (e.g. download interrupted after node was extracted but before npm) -/// would be reused forever and `npm_exec` could never self-heal. -fn probe_managed_install( - install_dir: &Path, - cache_root: &Path, - target_version: &str, -) -> Option { - if !install_dir.is_dir() { - return None; - } - // Canonicalise both sides so a symlink inside the install can't smuggle - // a repo-controlled tree past the `starts_with` check. `cache_root` must - // exist because the caller created `install_dir` under it, but be - // defensive: treat a failed canonicalize as "not trustworthy". - let canon_install = match std::fs::canonicalize(install_dir) { - Ok(p) => p, - Err(err) => { - tracing::warn!( - install_dir = %install_dir.display(), - error = %err, - "[node_runtime::bootstrap] canonicalize(install_dir) failed; treating as unusable" - ); - return None; - } - }; - let canon_cache = match std::fs::canonicalize(cache_root) { - Ok(p) => p, - Err(err) => { - tracing::warn!( - cache_root = %cache_root.display(), - error = %err, - "[node_runtime::bootstrap] canonicalize(cache_root) failed; treating managed install as unusable" - ); - return None; + if let Ok(mut cached) = self.cached.lock() { + *cached = Some(adapted.clone()); } - }; - if !canon_install.starts_with(&canon_cache) { - tracing::warn!( - install_dir = %canon_install.display(), - cache_root = %canon_cache.display(), - "[node_runtime::bootstrap] refusing to reuse managed install outside the resolved cache root (possible spoof)" - ); - return None; - } - let bin_dir = managed_bin_dir(install_dir); - let version = target_version.trim_start_matches('v').to_string(); - let resolved = build_resolved(bin_dir, version, NodeSource::Managed).ok()?; - if !resolved.npm_bin.is_file() { - tracing::warn!( - npm_bin = %resolved.npm_bin.display(), - "[node_runtime::bootstrap] managed install missing npm; forcing reinstall" - ); - return None; + Ok(adapted) } - Some(resolved) } #[cfg(test)] diff --git a/src/openhuman/runtime/node/downloader.rs b/src/openhuman/runtime/node/downloader.rs deleted file mode 100644 index 9ccd24ad79..0000000000 --- a/src/openhuman/runtime/node/downloader.rs +++ /dev/null @@ -1,280 +0,0 @@ -//! Node.js distribution downloader with SHASUMS256 verification. -//! -//! Resolves the right archive for the current OS/arch off nodejs.org, -//! streams it to a caller-supplied temp path, and validates the SHA-256 -//! against the official `SHASUMS256.txt` for the release. Keeps everything -//! in one place so the bootstrap caller only needs to know "download this -//! version, give me the bytes on disk". -//! -//! ## Security -//! -//! We **require** a SHA-256 match before returning success — a corrupted or -//! tampered archive is treated the same as a failed download and the file -//! is deleted. There is no opt-out; skills will run untrusted code inside -//! the resolved Node runtime, so the integrity check is load-bearing. - -use anyhow::{anyhow, bail, Context, Result}; -use reqwest::Client; -use sha2::{Digest, Sha256}; -use std::collections::HashMap; -use std::path::Path; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; - -/// Base URL for official Node.js release artifacts. -const NODEJS_DIST_BASE: &str = "https://nodejs.org/dist"; - -/// Describes a single downloadable Node.js distribution for the host triple. -#[derive(Debug, Clone)] -pub struct NodeDistribution { - /// Version string including the leading `v` (e.g. `v22.11.0`). - pub version: String, - /// Archive filename as it appears in `SHASUMS256.txt` - /// (e.g. `node-v22.11.0-darwin-arm64.tar.xz`). - pub archive_name: String, - /// Full download URL. - pub url: String, - /// Whether the archive is a zip (Windows) or tar.xz (everything else). - /// Drives which extraction path the caller invokes. - pub is_zip: bool, -} - -impl NodeDistribution { - /// Build the distribution descriptor for the current host OS/arch. - /// - /// Supported triples mirror the officially-prebuilt Node.js binaries: - /// - /// | OS | Arch | Archive suffix | - /// |----------|--------------------------------------|-----------------------------| - /// | macOS | aarch64, x86_64 | `-darwin-{arm64,x64}.tar.xz`| - /// | Linux | aarch64, x86_64, arm, armv7 | `-linux-{arm64,x64,armv7l}.tar.xz` | - /// | Windows | aarch64, x86_64 | `-win-{arm64,x64}.zip` | - /// - /// Everything else yields an error — the caller should surface it as a - /// "Node runtime unavailable on this host" message. - pub fn for_host(version: &str) -> Result { - let version = normalize_version(version); - let (suffix, is_zip) = host_archive_suffix()?; - let archive_name = format!("node-{version}-{suffix}"); - let url = format!("{NODEJS_DIST_BASE}/{version}/{archive_name}"); - tracing::debug!( - version = %version, - url = %url, - "[node_runtime::downloader] resolved distribution for host" - ); - Ok(Self { - version, - archive_name, - url, - is_zip, - }) - } -} - -/// Normalise a version string to the canonical `vX.Y.Z` form used by -/// nodejs.org. Config allows `22.11.0` or `v22.11.0`; we always emit the -/// `v`-prefixed variant because it is what appears in the URL path. -fn normalize_version(raw: &str) -> String { - let trimmed = raw.trim(); - if trimmed.starts_with('v') { - trimmed.to_string() - } else { - format!("v{trimmed}") - } -} - -/// Return `(archive_suffix, is_zip)` for the current host. The suffix omits -/// the `node-vX.Y.Z-` prefix because callers always interpolate the version. -fn host_archive_suffix() -> Result<(&'static str, bool)> { - let os = std::env::consts::OS; - let arch = std::env::consts::ARCH; - match (os, arch) { - ("macos", "aarch64") => Ok(("darwin-arm64.tar.xz", false)), - ("macos", "x86_64") => Ok(("darwin-x64.tar.xz", false)), - ("linux", "aarch64") => Ok(("linux-arm64.tar.xz", false)), - ("linux", "x86_64") => Ok(("linux-x64.tar.xz", false)), - ("linux", "arm") | ("linux", "armv7") => Ok(("linux-armv7l.tar.xz", false)), - ("windows", "aarch64") => Ok(("win-arm64.zip", true)), - ("windows", "x86_64") => Ok(("win-x64.zip", true)), - _ => Err(anyhow!( - "no prebuilt Node.js distribution for host {os}/{arch} — set node.enabled=false or install node manually" - )), - } -} - -/// Fetch `SHASUMS256.txt` for the release and return a -/// `archive_name -> sha256_hex` map. The hex digest is lowercase. -pub async fn fetch_shasums(client: &Client, version: &str) -> Result> { - let version = normalize_version(version); - let url = format!("{NODEJS_DIST_BASE}/{version}/SHASUMS256.txt"); - tracing::debug!(url = %url, "[node_runtime::downloader] fetching SHASUMS256.txt"); - - let body = client - .get(&url) - .send() - .await - .with_context(|| format!("GET {url}"))? - .error_for_status() - .with_context(|| format!("non-success status on {url}"))? - .text() - .await - .with_context(|| format!("reading body of {url}"))?; - - let map = parse_shasums(&body); - tracing::debug!( - entries = map.len(), - "[node_runtime::downloader] parsed SHASUMS256.txt" - ); - Ok(map) -} - -/// Parse the `SHASUMS256.txt` body into a lookup table. The format is one -/// entry per line: ` ` (two spaces). Unknown / blank -/// lines are skipped to be robust against trailing newlines or signature -/// blocks that may appear in future releases. -fn parse_shasums(body: &str) -> HashMap { - let mut out = HashMap::new(); - for line in body.lines() { - let mut parts = line.split_whitespace(); - let (Some(hash), Some(name)) = (parts.next(), parts.next()) else { - continue; - }; - if hash.len() == 64 && hash.chars().all(|c| c.is_ascii_hexdigit()) { - out.insert(name.to_string(), hash.to_ascii_lowercase()); - } - } - out -} - -/// Stream `dist.url` to `target_path`, computing the SHA-256 on the fly and -/// comparing against the digest supplied in `expected_sha256`. -/// -/// On mismatch or any I/O error the partial file at `target_path` is -/// removed — we never leave half-written / tampered archives on disk. -pub async fn download_distribution( - client: &Client, - dist: &NodeDistribution, - target_path: &Path, - expected_sha256: &str, -) -> Result<()> { - tracing::info!( - url = %dist.url, - target = %target_path.display(), - "[node_runtime::downloader] starting download" - ); - - if let Some(parent) = target_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .with_context(|| format!("creating cache dir {}", parent.display()))?; - } - - let mut response = client - .get(&dist.url) - .send() - .await - .with_context(|| format!("GET {}", dist.url))? - .error_for_status() - .with_context(|| format!("non-success status on {}", dist.url))?; - - let total_bytes = response.content_length(); - let mut file = File::create(target_path) - .await - .with_context(|| format!("creating {}", target_path.display()))?; - let mut hasher = Sha256::new(); - let mut written: u64 = 0; - - // Stream into `file`. On any chunk / write / flush failure we remove - // the partial file on disk so a retry starts clean and callers never - // see a half-written archive. - let stream_result: Result<()> = async { - while let Some(chunk) = response - .chunk() - .await - .with_context(|| format!("streaming {}", dist.url))? - { - hasher.update(&chunk); - file.write_all(&chunk) - .await - .with_context(|| format!("writing chunk to {}", target_path.display()))?; - written = written.saturating_add(chunk.len() as u64); - } - file.flush() - .await - .with_context(|| format!("flushing {}", target_path.display()))?; - Ok(()) - } - .await; - - drop(file); - - if let Err(err) = stream_result { - tracing::warn!( - target = %target_path.display(), - error = %err, - "[node_runtime::downloader] streaming failed — removing partial archive" - ); - let _ = tokio::fs::remove_file(target_path).await; - return Err(err); - } - - let actual_hex = hex::encode(hasher.finalize()); - let expected = expected_sha256.trim().to_ascii_lowercase(); - - if actual_hex != expected { - tracing::error!( - expected = %expected, - actual = %actual_hex, - target = %target_path.display(), - "[node_runtime::downloader] SHA-256 mismatch — deleting partial archive" - ); - let _ = tokio::fs::remove_file(target_path).await; - bail!( - "SHA-256 mismatch for {} (expected {expected}, got {actual_hex})", - dist.archive_name - ); - } - - tracing::info!( - target = %target_path.display(), - bytes = written, - total = ?total_bytes, - "[node_runtime::downloader] download complete, hash verified" - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn normalizes_version_with_and_without_prefix() { - assert_eq!(normalize_version("22.11.0"), "v22.11.0"); - assert_eq!(normalize_version("v22.11.0"), "v22.11.0"); - assert_eq!(normalize_version(" v22.11.0\n"), "v22.11.0"); - } - - #[test] - fn parses_shasums_text() { - let body = "\ -abc123def4567890abc123def4567890abc123def4567890abc123def4567890 node-v22.11.0-darwin-arm64.tar.xz -1111222233334444555566667777888899990000111122223333444455556666 node-v22.11.0-linux-x64.tar.xz -garbage line -BADHASHNOTHEX node-v22.11.0-win-x64.zip -"; - let map = parse_shasums(body); - assert_eq!(map.len(), 2); - assert_eq!( - map.get("node-v22.11.0-darwin-arm64.tar.xz").unwrap(), - "abc123def4567890abc123def4567890abc123def4567890abc123def4567890" - ); - } - - #[test] - fn distribution_for_host_returns_sensible_url() { - let dist = NodeDistribution::for_host("v22.11.0").expect("host supported in CI"); - assert!(dist.url.starts_with("https://nodejs.org/dist/v22.11.0/")); - assert!(dist.archive_name.starts_with("node-v22.11.0-")); - } -} diff --git a/src/openhuman/runtime/node/extractor.rs b/src/openhuman/runtime/node/extractor.rs deleted file mode 100644 index 2fc46a3c79..0000000000 --- a/src/openhuman/runtime/node/extractor.rs +++ /dev/null @@ -1,235 +0,0 @@ -//! Archive extraction for downloaded Node.js distributions. -//! -//! Handles both shapes that nodejs.org ships: -//! -//! * `.tar.xz` on macOS and Linux — decoded via `xz2` then unpacked through -//! the `tar` crate. -//! * `.zip` on Windows — unpacked through the `zip` crate. -//! -//! All archives are "single-rooted": they expand into one top-level folder -//! like `node-v22.11.0-darwin-arm64/`. We extract into a caller-supplied -//! staging directory, then return the absolute path of that inner folder so -//! the bootstrap layer can rename/move it into the cache atomically. -//! -//! Extraction is CPU/IO-bound and the underlying crates are synchronous, so -//! we wrap the real work in `tokio::task::spawn_blocking` to keep the -//! runtime responsive. - -use anyhow::{anyhow, Context, Result}; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; - -/// Extract `archive` into `extract_root` and return the absolute path of the -/// single top-level folder produced by the archive. -/// -/// `is_zip = true` selects the zip path, otherwise the tar.xz path runs. -/// On any error the caller should treat `extract_root` as contaminated and -/// remove it before retrying — we do not auto-clean because the caller -/// typically owns a fresh temp dir. -pub async fn extract_distribution( - archive: &Path, - extract_root: &Path, - is_zip: bool, -) -> Result { - let archive = archive.to_path_buf(); - let extract_root = extract_root.to_path_buf(); - - tracing::info!( - archive = %archive.display(), - extract_root = %extract_root.display(), - is_zip, - "[node_runtime::extractor] starting extraction" - ); - - tokio::task::spawn_blocking(move || -> Result { - fs::create_dir_all(&extract_root) - .with_context(|| format!("creating extract root {}", extract_root.display()))?; - - if is_zip { - extract_zip(&archive, &extract_root)?; - } else { - extract_tar_xz(&archive, &extract_root)?; - } - - let top_level = find_single_top_level(&extract_root)?; - tracing::info!( - top_level = %top_level.display(), - "[node_runtime::extractor] extraction complete" - ); - Ok(top_level) - }) - .await - .context("spawn_blocking join failure during extraction")? -} - -/// Extract a `.tar.xz` archive into `extract_root`. -fn extract_tar_xz(archive: &Path, extract_root: &Path) -> Result<()> { - let file = - File::open(archive).with_context(|| format!("opening archive {}", archive.display()))?; - let decoder = xz2::read::XzDecoder::new(file); - let mut tar = tar::Archive::new(decoder); - // `set_preserve_permissions(true)` is the default on Unix; we restate - // it so the `node` binary keeps its `+x` bit after extraction. - tar.set_preserve_permissions(true); - tar.set_overwrite(true); - tar.unpack(extract_root) - .with_context(|| format!("unpacking tar.xz into {}", extract_root.display()))?; - Ok(()) -} - -/// Extract a `.zip` archive into `extract_root`. Handles directory entries, -/// file entries, and restores Unix mode bits where present (no-op on -/// Windows hosts, which is where `.zip` actually matters). -fn extract_zip(archive: &Path, extract_root: &Path) -> Result<()> { - let file = - File::open(archive).with_context(|| format!("opening archive {}", archive.display()))?; - let mut zip = zip::ZipArchive::new(file) - .with_context(|| format!("opening zip archive {}", archive.display()))?; - - for i in 0..zip.len() { - let mut entry = zip - .by_index(i) - .with_context(|| format!("reading zip entry {i}"))?; - let Some(relative) = entry.enclosed_name() else { - tracing::warn!( - name = entry.name(), - "[node_runtime::extractor] skipping zip entry with unsafe path" - ); - continue; - }; - let out_path = extract_root.join(relative); - - if entry.is_dir() { - fs::create_dir_all(&out_path) - .with_context(|| format!("creating {}", out_path.display()))?; - } else { - if let Some(parent) = out_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("creating {}", parent.display()))?; - } - let mut out = File::create(&out_path) - .with_context(|| format!("creating {}", out_path.display()))?; - io::copy(&mut entry, &mut out) - .with_context(|| format!("writing {}", out_path.display()))?; - } - - #[cfg(unix)] - if let Some(mode) = entry.unix_mode() { - use std::os::unix::fs::PermissionsExt; - fs::set_permissions(&out_path, fs::Permissions::from_mode(mode)) - .with_context(|| format!("chmod {}", out_path.display()))?; - } - } - - Ok(()) -} - -/// Locate the single top-level directory inside `extract_root`. Node.js -/// archives always produce one root folder; anything else (multiple -/// entries, only files) is a contract violation from our side and we -/// surface it as an error rather than guessing. -fn find_single_top_level(extract_root: &Path) -> Result { - let mut entries = fs::read_dir(extract_root) - .with_context(|| format!("listing {}", extract_root.display()))? - .collect::, _>>() - .with_context(|| format!("reading entries of {}", extract_root.display()))?; - - // Stable order for deterministic logging. - entries.sort_by_key(|e| e.file_name()); - - let mut dirs: Vec = entries - .into_iter() - .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) - .map(|e| e.path()) - .collect(); - - match dirs.len() { - 1 => Ok(dirs.pop().unwrap()), - 0 => Err(anyhow!( - "expected one top-level folder under {}, found none", - extract_root.display() - )), - n => Err(anyhow!( - "expected one top-level folder under {}, found {n}: {:?}", - extract_root.display(), - dirs - )), - } -} - -/// Atomically move `staged` into place at `final_dest`. -/// -/// Strategy: -/// 1. If `final_dest` already exists, move it to a sibling `.old-` -/// path so we never lose a working install even if a later step fails. -/// 2. Rename `staged` -> `final_dest`. On the same filesystem this is a -/// single `rename(2)` and is atomic from the reader's perspective. -/// 3. Best-effort cleanup of the `.old-*` directory. -/// -/// Returns the `final_dest` path on success. -pub async fn atomic_install(staged: &Path, final_dest: &Path) -> Result { - let staged = staged.to_path_buf(); - let final_dest = final_dest.to_path_buf(); - - tokio::task::spawn_blocking(move || -> Result { - if let Some(parent) = final_dest.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("creating parent {}", parent.display()))?; - } - - let mut backup: Option = None; - if final_dest.exists() { - let ts = std::process::id(); - let candidate = final_dest.with_extension(format!("old-{ts}")); - fs::rename(&final_dest, &candidate).with_context(|| { - format!( - "moving existing install {} aside to {}", - final_dest.display(), - candidate.display() - ) - })?; - backup = Some(candidate); - } - - if let Err(err) = fs::rename(&staged, &final_dest).with_context(|| { - format!( - "renaming staged {} -> {}", - staged.display(), - final_dest.display() - ) - }) { - // Stage->final rename failed; restore the previous install from - // backup so the working runtime stays in place. Surface any - // restore failure separately (as a warning) but always return - // the original error. - if let Some(backup_path) = backup.as_ref() { - if let Err(restore_err) = fs::rename(backup_path, &final_dest) { - tracing::warn!( - backup = %backup_path.display(), - final_dest = %final_dest.display(), - error = %restore_err, - "[node_runtime::extractor] failed to restore backup after staged rename failure" - ); - } else { - tracing::info!( - final_dest = %final_dest.display(), - "[node_runtime::extractor] restored previous install after staged rename failure" - ); - } - } - return Err(err); - } - - if let Some(path) = backup { - let _ = fs::remove_dir_all(&path); - } - tracing::info!( - final_dest = %final_dest.display(), - "[node_runtime::extractor] atomic install complete" - ); - Ok(final_dest) - }) - .await - .context("spawn_blocking join failure during atomic install")? -} diff --git a/src/openhuman/runtime/node/resolver.rs b/src/openhuman/runtime/node/resolver.rs deleted file mode 100644 index a800c54ae8..0000000000 --- a/src/openhuman/runtime/node/resolver.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! System-node resolver. -//! -//! Walks `PATH`, probes `node --version`, and returns a [`SystemNode`] when -//! the host-installed binary matches the configured target major version. -//! Runs synchronously because it blocks on one short-lived subprocess and is -//! called exactly once per bootstrap — pushing it onto the Tokio runtime -//! would add noise without benefit. -//! -//! Target-version matching is intentionally loose: we only compare **major** -//! versions. Point releases of Node.js are ABI-stable, and skills pin their -//! own dependency versions via `package.json` / `package-lock.json`, so a -//! host `v22.8.0` is accepted when `node.version = "v22.11.0"`. If a user -//! needs strict pinning they can set `node.prefer_system = false`. - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::time::Duration; - -/// A usable Node.js toolchain discovered on the host `PATH`. -#[derive(Debug, Clone)] -pub struct SystemNode { - /// Absolute path to the `node` executable. - pub path: PathBuf, - /// Parsed major version (e.g. `22`). - pub major: u32, - /// Raw version string reported by `node --version`, trimmed of the - /// leading `v` and trailing whitespace (e.g. `"22.11.0"`). - pub version: String, -} - -/// Parse a version string like `v22.11.0` / `22.11.0` / `v22` and return the -/// numeric major component. -/// -/// Returns `None` when the input is malformed. Tolerant of surrounding -/// whitespace and an optional leading `v` prefix so it can accept both the -/// config value (`node.version = "v22.11.0"`) and the raw `node --version` -/// output (`v22.11.0\n`). -pub fn parse_node_version(raw: &str) -> Option { - let trimmed = raw.trim(); - let stripped = trimmed.strip_prefix('v').unwrap_or(trimmed); - let major = stripped.split('.').next()?; - major.parse::().ok() -} - -/// Probe the host for a `node` binary on `PATH` whose major version matches -/// `target_version`. Returns `Some(SystemNode)` on success, `None` when no -/// compatible toolchain is found. -/// -/// Heavy tracing is intentional — resolver decisions drive whether we skip a -/// multi-hundred-MB download, so operators need a clear breadcrumb trail. -pub fn detect_system_node(target_version: &str) -> Option { - let Some(target_major) = parse_node_version(target_version) else { - tracing::warn!( - target_version, - "[node_runtime::resolver] invalid target_version, skipping system-node probe" - ); - return None; - }; - - let Some(path) = which_node() else { - tracing::debug!( - "[node_runtime::resolver] no `node` found on PATH — will fall back to download" - ); - return None; - }; - - tracing::debug!( - path = %path.display(), - target_major, - "[node_runtime::resolver] probing system node" - ); - - let Some(version) = probe_node_version(&path) else { - tracing::warn!( - path = %path.display(), - "[node_runtime::resolver] `node --version` failed; treating as unavailable" - ); - return None; - }; - - let Some(host_major) = parse_node_version(&version) else { - tracing::warn!( - path = %path.display(), - version = %version, - "[node_runtime::resolver] could not parse `node --version` output" - ); - return None; - }; - - if host_major != target_major { - tracing::info!( - path = %path.display(), - host_major, - target_major, - "[node_runtime::resolver] host node major mismatch — will download managed runtime" - ); - return None; - } - - // `npm_exec` rides on the same resolved toolchain. On distros that - // package `nodejs` and `npm` separately (Debian/Ubuntu default, - // Alpine's `nodejs-current`, some NixOS setups) the `node` binary can - // be present without `npm`. If we cached `NodeSource::System` here - // every `npm_exec` call would break with an obscure error. Require a - // usable `npm --version` probe before accepting the system toolchain; - // on failure, return `None` so the managed download path takes over. - let Some(npm_path) = which_npm() else { - tracing::info!( - node_path = %path.display(), - "[node_runtime::resolver] compatible system node found but `npm` is missing on PATH — falling back to managed runtime" - ); - return None; - }; - - if probe_subcommand_version(&npm_path, "npm").is_none() { - tracing::warn!( - npm_path = %npm_path.display(), - "[node_runtime::resolver] `npm --version` failed; falling back to managed runtime" - ); - return None; - } - - let normalized = version.trim_start_matches('v').trim().to_string(); - tracing::info!( - path = %path.display(), - npm_path = %npm_path.display(), - version = %normalized, - "[node_runtime::resolver] reusing compatible system node (npm verified)" - ); - Some(SystemNode { - path, - major: host_major, - version: normalized, - }) -} - -/// Locate a `node` binary on `PATH`. Cross-platform: appends the host -/// executable suffix (`.exe` on Windows) so callers receive a path that can -/// be invoked directly. -/// -/// Unix command lookup skips non-executable entries. A non-executable -/// placeholder earlier in `PATH` (e.g. an unprivileged `node` shim left by -/// a failed install) would otherwise mask a valid later install and force -/// the managed runtime download. We mirror the shell behaviour by checking -/// the execute bit before returning. -fn which_node() -> Option { - let exe_name = format!("node{}", std::env::consts::EXE_SUFFIX); - which_exe(&exe_name) -} - -/// Locate an `npm` binary on `PATH`. Applies the same execute-bit filter -/// as [`which_node`]. On Windows we look for `npm.cmd` first (the official -/// installer ships a batch shim; there is no `npm.exe`) and fall back to -/// `npm` for unusual setups that expose a bare binary. -fn which_npm() -> Option { - #[cfg(windows)] - { - if let Some(p) = which_exe("npm.cmd") { - return Some(p); - } - which_exe("npm") - } - #[cfg(not(windows))] - { - which_exe("npm") - } -} - -/// `PATH` search helper shared by `which_node` / `which_npm`. Applies the -/// platform-specific executability check so a non-executable placeholder -/// earlier in `PATH` doesn't shadow a valid later entry. -fn which_exe(exe_name: &str) -> Option { - let path_var = std::env::var_os("PATH")?; - for dir in std::env::split_paths(&path_var) { - let candidate = dir.join(exe_name); - if is_executable_candidate(&candidate) { - return Some(candidate); - } - } - None -} - -#[cfg(unix)] -fn is_executable_candidate(path: &std::path::Path) -> bool { - use std::os::unix::fs::PermissionsExt; - std::fs::metadata(path) - .map(|meta| meta.is_file() && (meta.permissions().mode() & 0o111 != 0)) - .unwrap_or(false) -} - -#[cfg(not(unix))] -fn is_executable_candidate(path: &std::path::Path) -> bool { - // On Windows, the `.exe` suffix already encodes executability for the - // loader; any regular file matching `node.exe` is a valid candidate. - path.is_file() -} - -/// Invoke ` --version` with a real 5-second timeout and return the raw -/// version string on success. The timeout guards against a broken shim on -/// `PATH` hanging the bootstrap indefinitely. -fn probe_node_version(path: &std::path::Path) -> Option { - probe_subcommand_version(path, "node") -} - -/// Same semantics as [`probe_node_version`], but usable for arbitrary -/// toolchain binaries. `label` is only used for log attribution. -fn probe_subcommand_version(path: &std::path::Path, label: &str) -> Option { - use std::io::Read; - use wait_timeout::ChildExt; - - let mut cmd = Command::new(path); - cmd.arg("--version") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - #[cfg(windows)] - { - use std::os::windows::process::CommandExt; - cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW - } - let mut child = cmd.spawn().ok()?; - - let timeout = Duration::from_secs(5); - let status = match child.wait_timeout(timeout).ok()? { - Some(s) => s, - None => { - tracing::warn!( - path = %path.display(), - label, - timeout_secs = 5, - "[node_runtime::resolver] ` --version` timed out; killing process" - ); - let _ = child.kill(); - let _ = child.wait(); - return None; - } - }; - - if !status.success() { - let mut stderr_buf = String::new(); - if let Some(mut s) = child.stderr.take() { - let _ = s.read_to_string(&mut stderr_buf); - } - tracing::debug!( - status = ?status, - label, - stderr = %stderr_buf, - "[node_runtime::resolver] ` --version` exited non-zero" - ); - return None; - } - - let mut stdout_buf = String::new(); - if let Some(mut s) = child.stdout.take() { - let _ = s.read_to_string(&mut stdout_buf); - } - let trimmed = stdout_buf.trim().to_string(); - if trimmed.is_empty() { - return None; - } - Some(trimmed) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parses_version_with_v_prefix() { - assert_eq!(parse_node_version("v22.11.0"), Some(22)); - } - - #[test] - fn parses_version_without_v_prefix() { - assert_eq!(parse_node_version("22.11.0"), Some(22)); - } - - #[test] - fn parses_major_only() { - assert_eq!(parse_node_version("v22"), Some(22)); - } - - #[test] - fn tolerates_surrounding_whitespace() { - assert_eq!(parse_node_version(" v22.11.0\n"), Some(22)); - } - - #[test] - fn rejects_garbage() { - assert_eq!(parse_node_version("not-a-version"), None); - assert_eq!(parse_node_version(""), None); - assert_eq!(parse_node_version("v"), None); - } -} From 3663c55827b4eff8a79aec5bbf15f2428a30cdb2 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:08:36 +0300 Subject: [PATCH 13/55] refactor(node): move Node toolchain resolution into the bootstrap module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Node runtime module no longer owns downloading, extracting, or resolving system Node — those responsibilities now live in the `tinyruntime` module. The bootstrap module becomes a thin client that adapts a `ResolvedRuntime` into a `ResolvedNode`, and the test suite is rewritten to cover that adaptation seam instead of the old probe-and-install flow. The `downloader`, `extractor`, and `resolver` submodules are removed, and the module documentation is updated to reflect the new boundary. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/bootstrap_tests.rs | 146 ++++++++++-------- src/openhuman/runtime/node/mod.rs | 57 +++---- 2 files changed, 102 insertions(+), 101 deletions(-) diff --git a/src/openhuman/runtime/node/bootstrap_tests.rs b/src/openhuman/runtime/node/bootstrap_tests.rs index a0728ecc1f..55850f5bd1 100644 --- a/src/openhuman/runtime/node/bootstrap_tests.rs +++ b/src/openhuman/runtime/node/bootstrap_tests.rs @@ -1,84 +1,102 @@ -use super::*; +//! Tests for the Node toolchain client. +//! +//! What is worth testing here is the adaptation, not the resolution: the module +//! owns probing, downloading, verifying, and installing, and it has its own +//! suite for all of it. These cover the seam — how a module answer becomes a +//! [`ResolvedNode`], and what happens when the host has Node turned off. -fn touch(path: &Path) { - if let Some(parent) = path.parent() { - std::fs::create_dir_all(parent).unwrap(); - } - std::fs::write(path, b"#!/bin/sh\n").unwrap(); -} +use std::sync::Arc; + +use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeLayout, RuntimeSource}; -fn managed_config(cache_root: &Path) -> NodeConfig { - NodeConfig { - enabled: true, - version: NodeConfig::default().version, - cache_dir: cache_root.to_string_lossy().to_string(), - // Force the managed path so the probe never depends on a host node. - prefer_system: false, +use super::{NodeBootstrap, NodeSource, ResolvedNode}; +use crate::openhuman::config::Config; + +/// A module resolution carrying `executables`. +fn resolution(executables: &[(&str, &str)]) -> ResolvedRuntime { + let mut layout = RuntimeLayout::new("22.11.0", "/cache/node-v22.11.0/bin"); + for (name, path) in executables { + layout = layout.with_executable(*name, *path); } + ResolvedRuntime::from_layout(Language::nodejs(), RuntimeSource::Managed, layout) } -/// GH-5047: a warm restart is a fresh process, so the in-memory -/// `try_cached` memo is empty. The durable probe must still recover -/// readiness from the on-disk managed install — otherwise `is_done` reports -/// "not ready" every launch and the harness-init overlay re-appears. -#[tokio::test] -async fn probe_installed_true_from_disk_after_simulated_restart() { - let tmp = tempfile::tempdir().expect("tempdir"); - let cache_root = tmp.path(); +#[test] +fn a_resolution_becomes_the_paths_callers_name() { + let adapted = ResolvedNode::from_module(&resolution(&[ + ("node", "/cache/node-v22.11.0/bin/node"), + ("npm", "/cache/node-v22.11.0/bin/npm"), + ])) + .expect("a toolchain reporting node adapts"); - let config = managed_config(cache_root); - let dist = NodeDistribution::for_host(&config.version).expect("host arch supported"); - let bootstrap = NodeBootstrap::new(config, tmp.path().join("ws"), Client::new()); - - // Lay down a managed install exactly where the bootstrap expects it. - let bin_dir = managed_bin_dir(&bootstrap.install_dir(&dist)); - let (node_name, npm_name) = if cfg!(windows) { - ("node.exe", "npm.cmd") - } else { - ("node", "npm") - }; - touch(&bin_dir.join(node_name)); - touch(&bin_dir.join(npm_name)); + assert_eq!(adapted.node_bin, std::path::Path::new("/cache/node-v22.11.0/bin/node")); + assert_eq!(adapted.npm_bin, std::path::Path::new("/cache/node-v22.11.0/bin/npm")); + assert_eq!(adapted.bin_dir, std::path::Path::new("/cache/node-v22.11.0/bin")); + assert_eq!(adapted.version, "22.11.0"); + assert_eq!(adapted.source, NodeSource::Managed); +} - // Simulated cold process: nothing memoised yet. +#[test] +fn a_toolchain_without_npm_still_resolves() { + // Refusing here would take `node_exec` down along with `npm_exec`, for a + // toolchain that runs `node` perfectly well. + let adapted = ResolvedNode::from_module(&resolution(&[("node", "/usr/bin/node")])) + .expect("a toolchain without npm is still usable"); + assert_eq!(adapted.node_bin, std::path::Path::new("/usr/bin/node")); assert!( - bootstrap.try_cached().is_none(), - "precondition: process-local cache is empty right after a restart" + adapted.npm_bin.ends_with(if cfg!(windows) { "npm.cmd" } else { "npm" }), + "npm was not derived: {}", + adapted.npm_bin.display() ); +} - // The durable probe recovers readiness from disk (and never downloads). - assert!( - bootstrap.probe_installed().await.is_some(), - "durable probe should detect the on-disk managed node install" - ); - assert!( - bootstrap.try_cached().is_some(), - "a probe hit should memoise into the shared cache for the rest of the process" +#[test] +fn a_toolchain_without_node_is_refused() { + // Deriving `node` too would turn a broken install into a spawn failure much + // later, with nothing pointing at the cause. + let error = ResolvedNode::from_module(&resolution(&[("npm", "/usr/bin/npm")])) + .expect_err("a toolchain with no node is not a toolchain"); + assert!(error.to_string().contains("`node`"), "got `{error}`"); +} + +#[test] +fn the_version_never_keeps_a_leading_v() { + // Callers render this and compare it; `v22.11.0` and `22.11.0` reaching + // different call sites is exactly the drift the normalisation prevents. + let mut resolved = resolution(&[("node", "/usr/bin/node")]); + resolved.version = "v22.11.0".to_string(); + let adapted = ResolvedNode::from_module(&resolved).expect("adapts"); + assert_eq!(adapted.version, "22.11.0"); +} + +#[test] +fn a_system_toolchain_is_reported_as_one() { + let mut resolved = resolution(&[("node", "/usr/bin/node")]); + resolved.source = RuntimeSource::System; + assert_eq!( + ResolvedNode::from_module(&resolved).expect("adapts").source, + NodeSource::System ); } -/// A fresh machine (empty cache, no install) must report "not installed" so -/// a genuine first-run download still runs and the overlay still shows. #[tokio::test] -async fn probe_installed_none_when_nothing_on_disk() { - let tmp = tempfile::tempdir().expect("tempdir"); - let bootstrap = NodeBootstrap::new( - managed_config(tmp.path()), - tmp.path().join("ws"), - Client::new(), - ); +async fn a_disabled_runtime_refuses_before_reaching_the_bus() { + let mut config = Config::default(); + config.node.enabled = false; + let bootstrap = NodeBootstrap::new(Arc::new(config)); + + let error = bootstrap.resolve().await.expect_err("node is off"); + assert!(error.to_string().contains("disabled"), "got `{error}`"); assert!( bootstrap.probe_installed().await.is_none(), - "no on-disk install → provisioning still required" + "a disabled runtime has nothing provisioned" ); } -/// A disabled runtime is "nothing to provision", not "installed". -#[tokio::test] -async fn probe_installed_none_when_disabled() { - let tmp = tempfile::tempdir().expect("tempdir"); - let mut config = managed_config(tmp.path()); - config.enabled = false; - let bootstrap = NodeBootstrap::new(config, tmp.path().join("ws"), Client::new()); - assert!(bootstrap.probe_installed().await.is_none()); +#[test] +fn nothing_is_cached_before_the_first_resolution() { + // The shell consults this on every command; answering with a stale or + // invented toolchain would put the wrong directory on PATH. + let bootstrap = NodeBootstrap::new(Arc::new(Config::default())); + assert!(bootstrap.try_cached().is_none()); } diff --git a/src/openhuman/runtime/node/mod.rs b/src/openhuman/runtime/node/mod.rs index e39890ee00..f0575c8892 100644 --- a/src/openhuman/runtime/node/mod.rs +++ b/src/openhuman/runtime/node/mod.rs @@ -1,42 +1,31 @@ -//! Managed Node.js runtime and tool bridge. +//! Managed Node.js runtime and the generic tool bridge. //! -//! Responsibilities are split across submodules: +//! Two unrelated things share this directory, and the difference matters: +//! +//! * [`bootstrap`] is the *client* for the `tinyruntime` module. It asks for a +//! Node toolchain and adapts the answer. It downloads nothing, unpacks +//! nothing, and manages no cache — the module owns all of that now. +//! * [`ops`] and [`types`] are the generic native-tool dispatcher over the agent +//! tool registry (`oh:*` tools such as `memory_search`, file, and shell +//! tools). They have nothing to do with Node beyond being reachable from +//! JavaScript, which is why they are not gated below. //! -//! * [`resolver`] — detect a compatible system `node` on `PATH`. Cheap, -//! synchronous, called first so we can skip the download path when a -//! matching toolchain already exists on the host. -//! * [`bootstrap`] / [`downloader`] / [`extractor`] — resolve or install the -//! managed Node.js toolchain shipped with the core. -//! * [`ops`] / [`types`] — the generic runtime tool bridge: build / list / -//! classify / execute against the native agent tool registry. -//! * [`schemas`] / [`rpc`] — the gated `javascript.*` controller pair. - //! ## Gating (`runtime-node`) //! -//! Facade: this module is always declared, but only the *managed-Node* -//! machinery (`bootstrap` / `downloader` / `extractor` / `resolver` / `rpc` / -//! `schemas`) is `#[cfg(feature = "runtime-node")]`; a `stub` carries -//! `NodeBootstrap`'s type surface when the feature is off. The forcing -//! constraint is `ShellTool`, which holds `Option>` as a -//! field and is kernel — always compiled. +//! The module is always declared, but the managed-Node client and the +//! `javascript.*` controller pair are `#[cfg(feature = "runtime-node")]`; a +//! [`stub`] carries [`NodeBootstrap`]'s type surface when the feature is off. +//! The forcing constraint is `ShellTool`, which holds +//! `Option>` as a field and is kernel — always compiled. //! -//! [`ops`] and [`types`] are deliberately **not** gated. They are the generic -//! native-tool dispatcher over the agent tool registry (`oh:*` tools such as -//! `memory_search`, file, and shell tools) — they back both the gated -//! `javascript.*` controllers *and* the ungated `flows` `oh:` `NativeToolBackend`, -//! which must keep dispatching native tools even when the managed Node runtime -//! itself is compiled out. Only the JavaScript RPC and the Node-specific -//! `node_exec` / `npm_exec` tools are gated. +//! [`ops`] and [`types`] stay ungated because they back both the gated +//! `javascript.*` controllers *and* the ungated `flows` `oh:` backend, which +//! must keep dispatching native tools when the managed Node runtime is compiled +//! out. #[cfg(feature = "runtime-node")] pub mod bootstrap; #[cfg(feature = "runtime-node")] -pub mod downloader; -#[cfg(feature = "runtime-node")] -pub mod extractor; -#[cfg(feature = "runtime-node")] -pub mod resolver; -#[cfg(feature = "runtime-node")] pub mod rpc; #[cfg(feature = "runtime-node")] mod schemas; @@ -49,18 +38,12 @@ pub mod types; #[cfg(not(feature = "runtime-node"))] mod stub; #[cfg(not(feature = "runtime-node"))] -pub use stub::{NodeBootstrap, NodeSource, ResolvedNode, RUNTIME_NODE_DISABLED_MESSAGE}; +pub use stub::{NodeBootstrap, NodeSource, RUNTIME_NODE_DISABLED_MESSAGE, ResolvedNode}; #[cfg(feature = "runtime-node")] pub use bootstrap::{NodeBootstrap, NodeSource, ResolvedNode}; -#[cfg(feature = "runtime-node")] -pub use downloader::{download_distribution, fetch_shasums, NodeDistribution}; -#[cfg(feature = "runtime-node")] -pub use extractor::{atomic_install, extract_distribution}; pub use ops::{execute_tool, list_tools}; #[cfg(feature = "runtime-node")] -pub use resolver::{detect_system_node, parse_node_version, SystemNode}; -#[cfg(feature = "runtime-node")] pub use schemas::{ all_controller_schemas as all_runtime_node_controller_schemas, all_registered_controllers as all_runtime_node_registered_controllers, From 0fa2081f954e2405d8a8159dbb85ad34f7b0214c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:09:23 +0300 Subject: [PATCH 14/55] refactor(runtime): delegate Python interpreter resolution to the tinyruntime module The Python bootstrap no longer owns interpreter discovery, download, or installation. All of that work has moved into the shared `tinyruntime` module, which handles the same pipeline for every language. What remains in the bootstrap is the adapter that turns a module answer into the `ResolvedPython` type that callers already name, plus the `spawn_stdio` method that launches long-lived Python children. The old downloader, extractor, and resolver modules have been removed entirely, and the tests now cover only the seam between the module and the core. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/python/bootstrap.rs | 453 +++++------------- .../runtime/python/bootstrap_tests.rs | 270 +++-------- src/openhuman/runtime/python/downloader.rs | 275 ----------- .../runtime/python/downloader_tests.rs | 63 --- src/openhuman/runtime/python/extractor.rs | 113 ----- src/openhuman/runtime/python/mod.rs | 18 +- src/openhuman/runtime/python/resolver.rs | 277 ----------- .../runtime/python/resolver_tests.rs | 96 ---- 8 files changed, 201 insertions(+), 1364 deletions(-) delete mode 100644 src/openhuman/runtime/python/downloader.rs delete mode 100644 src/openhuman/runtime/python/downloader_tests.rs delete mode 100644 src/openhuman/runtime/python/extractor.rs delete mode 100644 src/openhuman/runtime/python/resolver.rs delete mode 100644 src/openhuman/runtime/python/resolver_tests.rs diff --git a/src/openhuman/runtime/python/bootstrap.rs b/src/openhuman/runtime/python/bootstrap.rs index 1fd8ab5e51..8dcc94d50c 100644 --- a/src/openhuman/runtime/python/bootstrap.rs +++ b/src/openhuman/runtime/python/bootstrap.rs @@ -1,380 +1,193 @@ -//! Python bootstrap orchestrator. +//! Python interpreter resolution, delegated to the `tinyruntime` module. //! -//! Resolves a managed standalone CPython distribution by default, with an -//! optional system-Python override for development. +//! This used to own interpreter discovery and a managed-CPython install +//! pipeline. Both now live in the `tinyruntime` module, which does the same work +//! for every language, so what is left is the adapter that turns a module answer +//! into the [`ResolvedPython`] this core's callers already name. +//! +//! # What still happens here +//! +//! [`spawn_stdio`](PythonBootstrap::spawn_stdio) launches a long-lived Python +//! child of this process — the runtime Python server, and the stdio MCP servers. +//! That is deliberately *not* the module's pooled execution: those children +//! outlive a single job, speak their own protocols, and are owned by the +//! subsystem that started them. The module resolves the interpreter; this core +//! decides what to run with it. + +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; -use anyhow::{bail, Context, Result}; -use reqwest::Client; -use std::fs::OpenOptions; -use std::path::{Path, PathBuf}; -use std::sync::Arc; -use tokio::sync::Mutex; +use anyhow::{Result, anyhow}; +use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; -use super::downloader::{download_distribution, select_distribution}; -use super::extractor::{atomic_install, extract_distribution}; -use super::resolver::{detect_system_python, SystemPython}; -use crate::openhuman::config::schema::RuntimePythonConfig; +use crate::openhuman::config::Config; +use crate::openhuman::modules::runtime; /// Origin of the resolved interpreter. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PythonSource { - /// Reused a compatible Python already available on the host. + /// Reused a compatible interpreter already on the host. System, - /// Reserved for a future managed CPython distribution. + /// A managed standalone distribution the module downloaded and installed. Managed, } +impl From for PythonSource { + fn from(source: RuntimeSource) -> Self { + match source { + RuntimeSource::System => Self::System, + // A source this build does not know is a module from a newer + // contract; Managed is the reading that treats it as provisioned. + _ => Self::Managed, + } + } +} + /// Fully-resolved Python interpreter. #[derive(Debug, Clone)] pub struct ResolvedPython { - /// Directory that should be prepended to `PATH` for child processes so - /// `python`, `python3`, `pip`, and `pip3` resolve to the same toolchain. - pub bin_dir: std::path::PathBuf, + /// Directory to prepend to `PATH` for child processes so `python`, + /// `python3`, and `pip` resolve to the same toolchain. + pub bin_dir: PathBuf, /// Absolute path to the Python executable. - pub python_bin: std::path::PathBuf, - /// Normalized interpreter version, e.g. `3.12.4`. + pub python_bin: PathBuf, + /// Normalised interpreter version, e.g. `3.12.4`. pub version: String, /// Where the interpreter came from. pub source: PythonSource, } -/// Serialised bootstrap entrypoint for Python runtime resolution. +impl ResolvedPython { + /// Adapt a module resolution, or say what it was missing. + fn from_module(resolved: &ResolvedRuntime) -> Result { + let python_bin = resolved + .executable("python") + .map(PathBuf::from) + .ok_or_else(|| anyhow!("the resolved python toolchain reports no interpreter"))?; + + Ok(Self { + bin_dir: PathBuf::from(&resolved.bin_dir), + python_bin, + version: resolved.version.clone(), + source: resolved.source.into(), + }) + } +} + +/// Resolves the Python interpreter through the `tinyruntime` module. pub struct PythonBootstrap { - config: RuntimePythonConfig, - client: Client, - cached: Arc>>, + config: Arc, + /// The last resolution, for the non-awaiting + /// [`PythonBootstrap::try_cached`]. + cached: Mutex>, } -impl PythonBootstrap { - pub fn new(config: RuntimePythonConfig) -> Self { - Self::new_with_client(config, Client::new()) +impl std::fmt::Debug for PythonBootstrap { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("PythonBootstrap") + .field("resolved", &self.try_cached().is_some()) + .finish_non_exhaustive() } +} - pub(crate) fn new_with_client(config: RuntimePythonConfig, client: Client) -> Self { +impl PythonBootstrap { + /// Build a bootstrap over this host's configuration. + #[must_use] + pub fn new(config: Arc) -> Self { Self { config, - client, - cached: Arc::new(Mutex::new(None)), + cached: Mutex::new(None), } } - /// Peek at the memoized interpreter without triggering a probe. - pub fn try_cached(&self) -> Option { - self.cached.try_lock().ok().and_then(|g| g.clone()) + /// The configuration this bootstrap resolves under. + #[must_use] + pub fn config(&self) -> &Config { + &self.config } - /// Resolve a Python 3.12+ interpreter. The first successful result is - /// memoized for subsequent callers. - pub async fn resolve(&self) -> Result { - let mut guard = self.cached.lock().await; - if let Some(existing) = guard.as_ref() { - tracing::debug!( - version = %existing.version, - source = ?existing.source, - "[runtime_python::bootstrap] returning cached ResolvedPython" - ); - return Ok(existing.clone()); - } - - if !self.config.enabled { - bail!( - "runtime_python is disabled (set runtime_python.enabled = true to use Python-backed integrations)" - ); - } - - if self.config.prefer_system { - if let Some(system) = detect_system_python( - &self.config.minimum_version, - empty_to_none(&self.config.preferred_command), - ) { - let resolved = resolve_from_system(system); - *guard = Some(resolved.clone()); - return Ok(resolved); - } - } - - let managed = self - .install_managed_from_api(super::downloader::RELEASES_API) - .await?; - *guard = Some(managed.clone()); - Ok(managed) + /// The memoised interpreter, without awaiting anything. + #[must_use] + pub fn try_cached(&self) -> Option { + self.cached.lock().ok().and_then(|guard| guard.clone()) } - /// Durable, **non-downloading** readiness probe. - /// - /// Unlike [`try_cached`], whose state is process-local and therefore empty - /// after every app restart, this inspects the host itself: a compatible - /// system interpreter (when `prefer_system`) or an already-extracted managed - /// distribution under the cache root. It performs only cheap `stat`/version - /// checks and **never** touches the network, so it is safe to call as a - /// warm-start guard before deciding whether to surface a visible - /// provisioning run. A hit is memoised into the same cache `resolve()` uses. - /// - /// Returns `Some(..)` when Python is already provisioned on disk, `None` - /// when a genuine download/install is still required (or the runtime is - /// disabled — callers treat that as "nothing to provision" separately). + /// Report an already-provisioned interpreter, without downloading one. pub async fn probe_installed(&self) -> Option { if let Some(existing) = self.try_cached() { return Some(existing); } - if !self.config.enabled { + if !self.config.runtime_python.enabled { return None; } - if self.config.prefer_system { - if let Some(system) = detect_system_python( - &self.config.minimum_version, - empty_to_none(&self.config.preferred_command), - ) { - let resolved = resolve_from_system(system); + match runtime::resolve(&self.config, &Language::python(), false).await { + Ok(Some(resolved)) => self.adopt(&resolved).ok(), + Ok(None) => { tracing::debug!( - version = %resolved.version, - "[runtime_python::bootstrap] durable probe found system python" + "[runtime::python] no python interpreter is provisioned yet (provisioning required)" ); - *self.cached.lock().await = Some(resolved.clone()); - return Some(resolved); + None } + Err(error) => { + tracing::debug!("[runtime::python] probing for an interpreter failed: {error}"); + None + } + } + } + + /// Resolve the interpreter, installing a managed one if the host has none. + /// + /// # Errors + /// + /// When Python is disabled for this host, when the module or its provider + /// cannot be loaded, or when provisioning fails. + pub async fn resolve(&self) -> Result { + if let Some(existing) = self.try_cached() { + return Ok(existing); } - if let Some(resolved) = probe_any_managed_install(&self.cache_root()) { - tracing::debug!( - version = %resolved.version, - "[runtime_python::bootstrap] durable probe found managed python on disk" - ); - *self.cached.lock().await = Some(resolved.clone()); - return Some(resolved); + if !self.config.runtime_python.enabled { + return Err(anyhow!( + "the python runtime is disabled (set runtime_python.enabled = true to use \ + python-backed integrations)" + )); } - tracing::debug!( - "[runtime_python::bootstrap] durable probe found no installed python (provisioning required)" - ); - None + + let resolved = runtime::resolve(&self.config, &Language::python(), true) + .await + .map_err(|error| anyhow!("{error}"))? + .ok_or_else(|| { + anyhow!("the python runtime module reported no interpreter and did not say why") + })?; + self.adopt(&resolved) } - /// Build a preconfigured child-process launcher for stdio-oriented Python - /// workloads such as MCP servers. + /// Launch a long-lived stdio Python child. + /// + /// # Errors + /// + /// When the interpreter cannot be resolved, or the child cannot be spawned. pub async fn spawn_stdio( &self, - spec: &crate::openhuman::runtime::python::process::PythonLaunchSpec, + spec: &super::process::PythonLaunchSpec, ) -> Result { let resolved = self.resolve().await?; - crate::openhuman::runtime::python::process::spawn_stdio_process(&resolved, spec) - } -} - -impl PythonBootstrap { - async fn install_managed(&self) -> Result { - self.install_managed_from_api(super::downloader::RELEASES_API) - .await + super::process::spawn_stdio_process(&resolved, spec) } - async fn install_managed_from_api(&self, releases_api_base: &str) -> Result { - let cache_root = self.cache_root(); - tokio::fs::create_dir_all(&cache_root) - .await - .with_context(|| format!("creating python runtime cache {}", cache_root.display()))?; - - let release = super::downloader::fetch_release_metadata_from_base( - &self.client, - releases_api_base, - empty_to_none(&self.config.managed_release_tag), - ) - .await?; - let dist = select_distribution( - &release, - &self.config.minimum_version, - &self.config.maximum_version, - )?; - let install_dir = cache_root.join(dist.install_dir_name()); - let _install_lock = acquire_install_lock(&install_dir).await?; - - if let Some(existing) = probe_managed_install(&install_dir) { - tracing::info!( - install_dir = %install_dir.display(), - version = %existing.version, - "[runtime_python::bootstrap] reusing existing managed python install" - ); - return Ok(existing); - } - + /// Adapt a module resolution and remember it. + fn adopt(&self, resolved: &ResolvedRuntime) -> Result { + let adapted = ResolvedPython::from_module(resolved)?; tracing::info!( - asset = %dist.asset_name, - release = %release.tag_name, - install_dir = %install_dir.display(), - "[runtime_python::bootstrap] installing managed python" + version = %adapted.version, + source = ?adapted.source, + "[runtime::python] python interpreter ready" ); - - let archive_path = cache_root.join(&dist.asset_name); - download_distribution(&self.client, &dist, &archive_path).await?; - - let scratch = cache_root.join(format!( - ".stage-{}-{}", - std::process::id(), - uuid::Uuid::new_v4() - )); - let _ = tokio::fs::remove_dir_all(&scratch).await; - let top_level = extract_distribution(&archive_path, &scratch).await?; - atomic_install(&top_level, &install_dir).await?; - let _ = tokio::fs::remove_dir_all(&scratch).await; - let _ = tokio::fs::remove_file(&archive_path).await; - - probe_managed_install(&install_dir).with_context(|| { - format!( - "managed python install completed but no interpreter was found under {}", - install_dir.display() - ) - }) - } - - fn cache_root(&self) -> PathBuf { - let configured = self.config.cache_dir.trim(); - if !configured.is_empty() { - return PathBuf::from(configured); - } - if let Some(user_cache) = dirs::cache_dir() { - return user_cache.join("openhuman").join("runtime-python"); - } - PathBuf::from(".openhuman").join("runtime-python") - } -} - -fn resolve_from_system(system: SystemPython) -> ResolvedPython { - tracing::info!( - path = %system.path.display(), - version = %system.version, - "[runtime_python::bootstrap] reusing compatible system python" - ); - ResolvedPython { - bin_dir: python_bin_dir(&system.path), - python_bin: system.path, - version: system.version, - source: PythonSource::System, - } -} - -fn empty_to_none(value: &str) -> Option<&str> { - let trimmed = value.trim(); - if trimmed.is_empty() { - None - } else { - Some(trimmed) - } -} - -/// Scan the managed cache root for an already-extracted CPython install, -/// probing each immediate subdirectory without any network access. Returns the -/// first usable interpreter found. Used by the durable readiness probe so a -/// warm restart detects a prior managed install without re-downloading — the -/// exact install-dir name is derived from network release metadata during -/// install, so on a cold cache we cannot reconstruct it offline and must scan. -fn probe_any_managed_install(cache_root: &Path) -> Option { - let entries = std::fs::read_dir(cache_root).ok()?; - for entry in entries.flatten() { - let path = entry.path(); - // Reject symlinked cache entries before probing: `path.is_dir()` follows - // symlinks, so a link planted under `cache_root` could point the durable - // probe at a directory *outside* the cache root and have it reused as a - // trusted managed install. `entry.file_type()` reports the link itself - // (it does not follow), so skip anything symlinked; treat an unknown type - // as untrusted too. (The Node bootstrap gets equivalent protection from - // its `canonicalize` + `starts_with(cache_root)` guard.) - match entry.file_type() { - Ok(ft) if ft.is_symlink() => continue, - Ok(_) => {} - Err(_) => continue, - } - if !path.is_dir() { - continue; - } - // Skip transient staging dirs (`.stage--`) left mid-install. - if path - .file_name() - .and_then(|n| n.to_str()) - .is_some_and(|n| n.starts_with(".stage-")) - { - continue; - } - if let Some(resolved) = probe_managed_install(&path) { - return Some(resolved); - } - } - None -} - -fn probe_managed_install(install_dir: &Path) -> Option { - let python_bin = find_python_binary(install_dir)?; - let version = super::resolver::probe_python_version_public(&python_bin)?; - let version_info = super::resolver::parse_python_version(&version)?; - Some(ResolvedPython { - bin_dir: python_bin_dir(&python_bin), - python_bin, - version: version_info.display(), - source: PythonSource::Managed, - }) -} - -fn python_bin_dir(python_bin: &Path) -> PathBuf { - python_bin - .parent() - .map(Path::to_path_buf) - .unwrap_or_else(|| PathBuf::from(".")) -} - -fn find_python_binary(install_dir: &Path) -> Option { - let candidates = [ - install_dir.join("bin").join("python3.12"), - install_dir.join("bin").join("python3"), - install_dir.join("bin").join("python"), - install_dir.join("python.exe"), - install_dir.join("python3.12.exe"), - install_dir.join("python3.exe"), - ]; - for candidate in candidates { - if candidate.is_file() { - return Some(candidate); - } - } - - for entry in walkdir::WalkDir::new(install_dir).into_iter().flatten() { - let path = entry.path(); - if !path.is_file() { - continue; - } - let Some(name) = path.file_name().and_then(|n| n.to_str()) else { - continue; - }; - if matches!( - name, - "python" | "python3" | "python3.12" | "python.exe" | "python3.exe" | "python3.12.exe" - ) { - return Some(path.to_path_buf()); + if let Ok(mut cached) = self.cached.lock() { + *cached = Some(adapted.clone()); } + Ok(adapted) } - None -} - -async fn acquire_install_lock(install_dir: &Path) -> Result { - let lock_path = install_dir.with_extension("lock"); - if let Some(parent) = lock_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .with_context(|| format!("creating lock parent {}", parent.display()))?; - } - - let lock_path_for_task = lock_path.clone(); - tokio::task::spawn_blocking(move || -> Result { - use fs2::FileExt; - - let file = OpenOptions::new() - .create(true) - .truncate(false) - .read(true) - .write(true) - .open(&lock_path_for_task) - .with_context(|| format!("opening install lock {}", lock_path_for_task.display()))?; - file.lock_exclusive() - .with_context(|| format!("locking install target {}", lock_path_for_task.display()))?; - Ok(file) - }) - .await - .context("join failure while acquiring runtime_python install lock")? } #[cfg(test)] diff --git a/src/openhuman/runtime/python/bootstrap_tests.rs b/src/openhuman/runtime/python/bootstrap_tests.rs index d97afbac0c..c72b41a7d6 100644 --- a/src/openhuman/runtime/python/bootstrap_tests.rs +++ b/src/openhuman/runtime/python/bootstrap_tests.rs @@ -1,230 +1,80 @@ -use super::*; -use axum::extract::State; -use axum::http::{HeaderValue, StatusCode}; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use serde_json::json; -use sha2::{Digest, Sha256}; -use std::sync::Arc; -use tokio::net::TcpListener; - -#[cfg(unix)] -#[tokio::test] -async fn install_managed_from_mock_astral_release_downloads_and_resolves_executable() { - use std::os::unix::fs::PermissionsExt; - - let archive_bytes = build_test_python_archive().expect("archive bytes"); - let archive_sha = hex::encode(Sha256::digest(&archive_bytes)); - let asset_name = test_asset_name(); - - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let addr = listener.local_addr().expect("local addr"); - - let app_state = Arc::new(MockAstralState { - asset_name: asset_name.to_string(), - archive_bytes, - archive_sha, - artifact_url: format!("http://{addr}/artifacts/{asset_name}"), - }); - - let app = Router::new() - .route("/releases/latest", get(mock_release_latest)) - .route("/artifacts/{asset}", get(mock_artifact)) - .with_state(app_state.clone()); - - let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); - - let tmp = tempfile::tempdir().expect("tempdir"); - let mut cfg = RuntimePythonConfig::default(); - cfg.cache_dir = tmp.path().join("runtime-python").display().to_string(); - cfg.managed_release_tag = String::new(); - cfg.prefer_system = false; - - let bootstrap = PythonBootstrap::new_with_client(cfg, reqwest::Client::new()); - let api_base = format!("http://{addr}/releases"); - - let resolved = bootstrap - .install_managed_from_api(&api_base) - .await - .expect("managed install should succeed"); +//! Tests for the Python interpreter client. +//! +//! The module owns discovery, selection, download, and install, and tests all of +//! it in its own repository. These cover the seam: how a module answer becomes a +//! [`ResolvedPython`], and what happens when the host has Python turned off. - assert_eq!(resolved.source, PythonSource::Managed); - assert!(resolved.python_bin.is_file(), "python binary should exist"); - - let mode = std::fs::metadata(&resolved.python_bin) - .expect("metadata") - .permissions() - .mode(); - assert_ne!(mode & 0o111, 0, "python binary must be executable"); - - let version = crate::openhuman::runtime::python::resolver::probe_python_version_public( - &resolved.python_bin, - ) - .expect("version probe"); - assert_eq!(version.trim(), "Python 3.12.13"); - - server.abort(); -} - -#[derive(Clone)] -struct MockAstralState { - asset_name: String, - archive_bytes: Vec, - archive_sha: String, - artifact_url: String, -} - -async fn mock_release_latest(State(state): State>) -> impl IntoResponse { - Json(json!({ - "tag_name": "20260510", - "assets": [ - { - "name": state.asset_name, - "browser_download_url": state.artifact_url, - "digest": format!("sha256:{}", state.archive_sha), - } - ] - })) -} - -async fn mock_artifact( - State(state): State>, - axum::extract::Path(asset): axum::extract::Path, -) -> impl IntoResponse { - if asset != state.asset_name { - return (StatusCode::NOT_FOUND, Vec::new()).into_response(); - } - - ( - [( - axum::http::header::CONTENT_TYPE, - HeaderValue::from_static("application/gzip"), - )], - state.archive_bytes.clone(), - ) - .into_response() -} - -#[cfg(unix)] -fn build_test_python_archive() -> anyhow::Result> { - use flate2::write::GzEncoder; - use flate2::Compression; - use tar::{Builder, Header}; - - let mut tar_bytes = Vec::new(); - { - let encoder = GzEncoder::new(&mut tar_bytes, Compression::default()); - let mut builder = Builder::new(encoder); - - let root = test_asset_name().trim_end_matches(".tar.gz"); - let bin_dir = format!("{root}/bin"); - let python_path = format!("{bin_dir}/python3.12"); - - let mut root_header = Header::new_gnu(); - root_header.set_entry_type(tar::EntryType::Directory); - root_header.set_mode(0o755); - root_header.set_size(0); - root_header.set_cksum(); - builder.append_data(&mut root_header, root, std::io::empty())?; +use std::sync::Arc; - let mut bin_header = Header::new_gnu(); - bin_header.set_entry_type(tar::EntryType::Directory); - bin_header.set_mode(0o755); - bin_header.set_size(0); - bin_header.set_cksum(); - builder.append_data(&mut bin_header, &bin_dir, std::io::empty())?; +use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeLayout, RuntimeSource}; - let script = b"#!/bin/sh\necho 'Python 3.12.13'\n"; - let mut python_header = Header::new_gnu(); - python_header.set_entry_type(tar::EntryType::Regular); - python_header.set_mode(0o755); - python_header.set_size(script.len() as u64); - python_header.set_cksum(); - builder.append_data(&mut python_header, &python_path, &script[..])?; +use super::{PythonBootstrap, PythonSource, ResolvedPython}; +use crate::openhuman::config::Config; - builder.into_inner()?.finish()?; +/// A module resolution carrying `executables`. +fn resolution(executables: &[(&str, &str)]) -> ResolvedRuntime { + let mut layout = RuntimeLayout::new("3.12.4", "/cache/cpython-3.12.4/python/bin"); + for (name, path) in executables { + layout = layout.with_executable(*name, *path); } - Ok(tar_bytes) + ResolvedRuntime::from_layout(Language::python(), RuntimeSource::Managed, layout) } -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -fn test_asset_name() -> &'static str { - "cpython-3.12.13+20260510-aarch64-apple-darwin-install_only.tar.gz" -} - -#[cfg(all(target_os = "macos", target_arch = "x86_64"))] -fn test_asset_name() -> &'static str { - "cpython-3.12.13+20260510-x86_64-apple-darwin-install_only.tar.gz" +#[test] +fn a_resolution_becomes_the_paths_callers_name() { + let adapted = ResolvedPython::from_module(&resolution(&[ + ("python", "/cache/cpython-3.12.4/python/bin/python3"), + ("pip", "/cache/cpython-3.12.4/python/bin/pip3"), + ])) + .expect("a toolchain reporting an interpreter adapts"); + + assert_eq!( + adapted.python_bin, + std::path::Path::new("/cache/cpython-3.12.4/python/bin/python3") + ); + assert_eq!(adapted.version, "3.12.4"); + assert_eq!(adapted.source, PythonSource::Managed); } -#[cfg(all(target_os = "linux", target_arch = "x86_64"))] -fn test_asset_name() -> &'static str { - "cpython-3.12.13+20260510-x86_64-unknown-linux-gnu-install_only.tar.gz" +#[test] +fn an_install_without_an_interpreter_is_refused() { + // Unlike npm for Node, there is nothing to fall back to: the interpreter is + // the toolchain. + let error = ResolvedPython::from_module(&resolution(&[("pip", "/usr/bin/pip3")])) + .expect_err("an install with no interpreter is not one"); + assert!(error.to_string().contains("interpreter"), "got `{error}`"); } -#[cfg(all(target_os = "linux", target_arch = "aarch64"))] -fn test_asset_name() -> &'static str { - "cpython-3.12.13+20260510-aarch64-unknown-linux-gnu-install_only.tar.gz" +#[test] +fn an_install_without_pip_still_resolves() { + let adapted = ResolvedPython::from_module(&resolution(&[("python", "/usr/bin/python3")])) + .expect("an interpreter without pip is still an interpreter"); + assert_eq!(adapted.python_bin, std::path::Path::new("/usr/bin/python3")); } -/// GH-5047: a warm restart is a fresh process, so `try_cached` is empty. The -/// durable probe must still recover readiness from a prior managed install on -/// disk — otherwise `is_done` reports "not ready" every launch and the -/// harness-init overlay re-appears. -#[cfg(unix)] -#[tokio::test] -async fn probe_installed_true_from_disk_after_simulated_restart() { - use std::os::unix::fs::PermissionsExt; - - let tmp = tempfile::tempdir().expect("tempdir"); - let cache_root = tmp.path().join("runtime-python"); - // A prior managed install, extracted under the cache root. - let bin_dir = cache_root.join("cpython-3.12.13-managed").join("bin"); - std::fs::create_dir_all(&bin_dir).expect("mkdir install"); - let python_path = bin_dir.join("python3.12"); - std::fs::write(&python_path, b"#!/bin/sh\necho 'Python 3.12.13'\n").expect("write python stub"); - std::fs::set_permissions(&python_path, std::fs::Permissions::from_mode(0o755)).expect("chmod"); - - let mut cfg = RuntimePythonConfig::default(); - cfg.enabled = true; - cfg.cache_dir = cache_root.display().to_string(); - cfg.prefer_system = false; // force the managed-scan path — no host dependency - - let bootstrap = PythonBootstrap::new(cfg); - assert!( - bootstrap.try_cached().is_none(), - "precondition: process-local cache is empty right after a restart" - ); - let resolved = bootstrap.probe_installed().await; - assert!( - resolved.is_some(), - "durable probe should recover the managed python install from disk" +#[test] +fn a_system_interpreter_is_reported_as_one() { + let mut resolved = resolution(&[("python", "/usr/bin/python3")]); + resolved.source = RuntimeSource::System; + assert_eq!( + ResolvedPython::from_module(&resolved).expect("adapts").source, + PythonSource::System ); - assert_eq!(resolved.unwrap().source, PythonSource::Managed); } -/// A fresh machine (empty cache, no install) must report "not installed" so a -/// genuine first-run download still runs and the overlay still shows. #[tokio::test] -async fn probe_installed_none_when_nothing_on_disk() { - let tmp = tempfile::tempdir().expect("tempdir"); - let mut cfg = RuntimePythonConfig::default(); - cfg.enabled = true; - cfg.cache_dir = tmp.path().join("runtime-python").display().to_string(); - cfg.prefer_system = false; - let bootstrap = PythonBootstrap::new(cfg); - assert!( - bootstrap.probe_installed().await.is_none(), - "no on-disk install → provisioning still required" - ); -} +async fn a_disabled_runtime_refuses_before_reaching_the_bus() { + let mut config = Config::default(); + config.runtime_python.enabled = false; + let bootstrap = PythonBootstrap::new(Arc::new(config)); -/// A disabled runtime is "nothing to provision", not "installed". -#[tokio::test] -async fn probe_installed_none_when_disabled() { - let mut cfg = RuntimePythonConfig::default(); - cfg.enabled = false; - let bootstrap = PythonBootstrap::new(cfg); + let error = bootstrap.resolve().await.expect_err("python is off"); + assert!(error.to_string().contains("disabled"), "got `{error}`"); assert!(bootstrap.probe_installed().await.is_none()); } + +#[test] +fn nothing_is_cached_before_the_first_resolution() { + let bootstrap = PythonBootstrap::new(Arc::new(Config::default())); + assert!(bootstrap.try_cached().is_none()); +} diff --git a/src/openhuman/runtime/python/downloader.rs b/src/openhuman/runtime/python/downloader.rs deleted file mode 100644 index 507ebffde2..0000000000 --- a/src/openhuman/runtime/python/downloader.rs +++ /dev/null @@ -1,275 +0,0 @@ -//! Managed standalone Python distribution downloader. -//! -//! Pulls release metadata from `astral-sh/python-build-standalone`, selects a -//! host-compatible `install_only` archive satisfying the configured minimum -//! Python version, downloads it, and verifies the published SHA-256 digest. - -use anyhow::{anyhow, bail, Context, Result}; -use reqwest::Client; -use serde::Deserialize; -use sha2::{Digest, Sha256}; -use std::path::Path; -use tokio::fs::File; -use tokio::io::AsyncWriteExt; - -use super::resolver::{parse_python_version, PythonVersion}; - -pub(crate) const RELEASES_API: &str = - "https://api.github.com/repos/astral-sh/python-build-standalone/releases"; - -#[derive(Debug, Clone, Deserialize)] -pub struct GithubRelease { - pub tag_name: String, - pub assets: Vec, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct GithubAsset { - pub name: String, - pub browser_download_url: String, - pub digest: Option, -} - -#[derive(Debug, Clone)] -pub struct PythonDistribution { - pub release_tag: String, - pub asset_name: String, - pub url: String, - pub version: PythonVersion, - pub expected_sha256: Option, -} - -impl PythonDistribution { - pub fn install_dir_name(&self) -> String { - self.asset_name.trim_end_matches(".tar.gz").to_string() - } -} - -pub async fn fetch_release_metadata( - client: &Client, - release_tag: Option<&str>, -) -> Result { - fetch_release_metadata_from_base(client, RELEASES_API, release_tag).await -} - -pub(crate) async fn fetch_release_metadata_from_base( - client: &Client, - releases_api_base: &str, - release_tag: Option<&str>, -) -> Result { - let url = if let Some(tag) = release_tag { - format!("{releases_api_base}/tags/{tag}") - } else { - format!("{releases_api_base}/latest") - }; - - tracing::debug!(url = %url, "[runtime_python::downloader] fetching release metadata"); - - client - .get(&url) - .header(reqwest::header::USER_AGENT, "openhuman-core/runtime_python") - .send() - .await - .with_context(|| format!("GET {url}"))? - .error_for_status() - .with_context(|| format!("non-success status on {url}"))? - .json::() - .await - .with_context(|| format!("decoding release metadata from {url}")) -} - -pub fn select_distribution( - release: &GithubRelease, - minimum_version: &str, - maximum_version: &str, -) -> Result { - let Some(minimum) = parse_python_version(minimum_version) else { - bail!("invalid runtime_python.minimum_version `{minimum_version}`"); - }; - // Empty string disables the upper bound. A non-empty but unparseable value - // is a config error worth surfacing rather than silently ignoring. - let maximum = if maximum_version.trim().is_empty() { - None - } else { - match parse_python_version(maximum_version) { - Some(v) => Some(v), - None => bail!("invalid runtime_python.maximum_version `{maximum_version}`"), - } - }; - let target_suffix = host_asset_suffix()?; - - let mut candidates = release - .assets - .iter() - .filter_map(|asset| parse_distribution_asset(asset, &release.tag_name)) - .filter(|dist| asset_matches_target(&dist.asset_name, target_suffix)) - .filter(|dist| dist.version >= minimum) - // Exclusive upper bound — keeps selection off newer pre-release series - // (e.g. 3.15.x betas, which parse as a bare `3.15.0`). - .filter(|dist| maximum.as_ref().is_none_or(|max| dist.version < *max)) - .collect::>(); - - if candidates.is_empty() { - bail!( - "no managed python-build-standalone asset found for host suffix `{target_suffix}` with version >= {}{} in release {}", - minimum.display(), - maximum - .as_ref() - .map(|m| format!(" and < {}", m.display())) - .unwrap_or_default(), - release.tag_name - ); - } - - candidates.sort_by(|a, b| { - b.version - .cmp(&a.version) - .then_with(|| a.asset_name.cmp(&b.asset_name)) - }); - - if let Some(preferred) = candidates - .iter() - .find(|dist| dist.asset_name.contains("install_only_stripped")) - .cloned() - { - return Ok(preferred); - } - - candidates - .into_iter() - .next() - .ok_or_else(|| anyhow!("internal error selecting managed python asset")) -} - -fn parse_distribution_asset(asset: &GithubAsset, release_tag: &str) -> Option { - let name = asset.name.as_str(); - if !name.starts_with("cpython-") || !name.ends_with(".tar.gz") || !name.contains("install_only") - { - return None; - } - - let rest = name.strip_prefix("cpython-")?; - let version_str = rest.split('+').next()?; - let version = parse_python_version(version_str)?; - - let expected_sha256 = asset - .digest - .as_deref() - .and_then(|digest| digest.strip_prefix("sha256:")) - .map(str::to_string); - - Some(PythonDistribution { - release_tag: release_tag.to_string(), - asset_name: asset.name.clone(), - url: asset.browser_download_url.clone(), - version, - expected_sha256, - }) -} - -fn host_asset_suffix() -> Result<&'static str> { - let os = std::env::consts::OS; - let arch = std::env::consts::ARCH; - match (os, arch) { - ("macos", "aarch64") => Ok("aarch64-apple-darwin-install_only.tar.gz"), - ("macos", "x86_64") => Ok("x86_64-apple-darwin-install_only.tar.gz"), - ("linux", "aarch64") => Ok("aarch64-unknown-linux-gnu-install_only.tar.gz"), - ("linux", "x86_64") => Ok("x86_64-unknown-linux-gnu-install_only.tar.gz"), - ("windows", "aarch64") => Ok("aarch64-pc-windows-msvc-install_only.tar.gz"), - ("windows", "x86_64") => Ok("x86_64-pc-windows-msvc-install_only.tar.gz"), - _ => Err(anyhow!( - "no managed standalone Python distribution for host {os}/{arch}" - )), - } -} - -fn asset_matches_target(asset_name: &str, target_suffix: &str) -> bool { - asset_name.ends_with(target_suffix) - || asset_name.ends_with( - &target_suffix.replace("-install_only.tar.gz", "-install_only_stripped.tar.gz"), - ) -} - -pub async fn download_distribution( - client: &Client, - dist: &PythonDistribution, - target_path: &Path, -) -> Result<()> { - tracing::info!( - url = %dist.url, - target = %target_path.display(), - "[runtime_python::downloader] starting download" - ); - - if let Some(parent) = target_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .with_context(|| format!("creating cache dir {}", parent.display()))?; - } - - let mut response = client - .get(&dist.url) - .header(reqwest::header::USER_AGENT, "openhuman-core/runtime_python") - .send() - .await - .with_context(|| format!("GET {}", dist.url))? - .error_for_status() - .with_context(|| format!("non-success status on {}", dist.url))?; - - let mut file = File::create(target_path) - .await - .with_context(|| format!("creating {}", target_path.display()))?; - let mut hasher = Sha256::new(); - - let stream_result: Result<()> = async { - while let Some(chunk) = response - .chunk() - .await - .with_context(|| format!("streaming {}", dist.url))? - { - hasher.update(&chunk); - file.write_all(&chunk) - .await - .with_context(|| format!("writing chunk to {}", target_path.display()))?; - } - file.flush() - .await - .with_context(|| format!("flushing {}", target_path.display()))?; - Ok(()) - } - .await; - - drop(file); - - if let Err(err) = stream_result { - let _ = tokio::fs::remove_file(target_path).await; - return Err(err); - } - - if let Some(expected) = dist.expected_sha256.as_deref() { - let actual_hex = hex::encode(hasher.finalize()); - if actual_hex != expected { - let _ = tokio::fs::remove_file(target_path).await; - bail!( - "SHA-256 mismatch for {} (expected {expected}, got {actual_hex})", - dist.asset_name - ); - } - } else { - tracing::warn!( - asset = %dist.asset_name, - "[runtime_python::downloader] release metadata did not include a digest; skipping SHA-256 verification" - ); - } - - tracing::info!( - target = %target_path.display(), - asset = %dist.asset_name, - "[runtime_python::downloader] download complete" - ); - Ok(()) -} - -#[cfg(test)] -#[path = "downloader_tests.rs"] -mod tests; diff --git a/src/openhuman/runtime/python/downloader_tests.rs b/src/openhuman/runtime/python/downloader_tests.rs deleted file mode 100644 index c2dcede3ee..0000000000 --- a/src/openhuman/runtime/python/downloader_tests.rs +++ /dev/null @@ -1,63 +0,0 @@ -use super::*; - -#[test] -fn parses_asset_into_distribution() { - let asset = GithubAsset { - name: "cpython-3.12.13+20260510-x86_64-apple-darwin-install_only.tar.gz".to_string(), - browser_download_url: "https://example.invalid/python.tar.gz".to_string(), - digest: Some("sha256:abc123".to_string()), - }; - let dist = parse_distribution_asset(&asset, "20260510").expect("dist"); - assert_eq!(dist.release_tag, "20260510"); - assert_eq!(dist.version.display(), "3.12.13"); - assert_eq!(dist.expected_sha256.as_deref(), Some("abc123")); -} - -#[test] -fn ignores_non_install_only_assets() { - let asset = GithubAsset { - name: "cpython-3.12.13+20260510-x86_64-apple-darwin-full.tar.zst".to_string(), - browser_download_url: "https://example.invalid/python.tar.zst".to_string(), - digest: None, - }; - assert!(parse_distribution_asset(&asset, "20260510").is_none()); -} - -/// Build a release with one `install_only` asset per supplied version, named -/// for the current host so `select_distribution` accepts them. -fn release_with_versions(versions: &[&str]) -> GithubRelease { - let suffix = host_asset_suffix().expect("host suffix"); - let assets = versions - .iter() - .map(|v| GithubAsset { - name: format!("cpython-{v}+20260623-{suffix}"), - browser_download_url: format!("https://example.invalid/{v}.tar.gz"), - digest: None, - }) - .collect(); - GithubRelease { - tag_name: "20260623".to_string(), - assets, - } -} - -#[test] -fn maximum_version_caps_selection_to_stable_series() { - // 3.15.0b3 parses to a bare 3.15.0 — the cap is what keeps us off it. - let release = release_with_versions(&["3.12.13", "3.13.5", "3.15.0"]); - let dist = select_distribution(&release, "3.12.0", "3.14.0").expect("dist"); - assert_eq!(dist.version.display(), "3.13.5"); -} - -#[test] -fn empty_maximum_version_disables_the_cap() { - let release = release_with_versions(&["3.13.5", "3.15.0"]); - let dist = select_distribution(&release, "3.12.0", "").expect("dist"); - assert_eq!(dist.version.display(), "3.15.0"); -} - -#[test] -fn invalid_maximum_version_is_an_error() { - let release = release_with_versions(&["3.13.5"]); - assert!(select_distribution(&release, "3.12.0", "not-a-version").is_err()); -} diff --git a/src/openhuman/runtime/python/extractor.rs b/src/openhuman/runtime/python/extractor.rs deleted file mode 100644 index d1817a356e..0000000000 --- a/src/openhuman/runtime/python/extractor.rs +++ /dev/null @@ -1,113 +0,0 @@ -//! Archive extraction for managed standalone Python distributions. - -use anyhow::{anyhow, Context, Result}; -use std::fs::{self, File}; -use std::path::{Path, PathBuf}; - -pub async fn extract_distribution(archive: &Path, extract_root: &Path) -> Result { - let archive = archive.to_path_buf(); - let extract_root = extract_root.to_path_buf(); - - tracing::info!( - archive = %archive.display(), - extract_root = %extract_root.display(), - "[runtime_python::extractor] starting extraction" - ); - - tokio::task::spawn_blocking(move || -> Result { - fs::create_dir_all(&extract_root) - .with_context(|| format!("creating extract root {}", extract_root.display()))?; - - let file = File::open(&archive) - .with_context(|| format!("opening archive {}", archive.display()))?; - let decoder = flate2::read::GzDecoder::new(file); - let mut tar = tar::Archive::new(decoder); - tar.set_preserve_permissions(true); - tar.set_overwrite(true); - tar.unpack(&extract_root) - .with_context(|| format!("unpacking tar.gz into {}", extract_root.display()))?; - - find_single_top_level(&extract_root) - }) - .await - .context("spawn_blocking join failure during extraction")? -} - -fn find_single_top_level(extract_root: &Path) -> Result { - let mut entries = fs::read_dir(extract_root) - .with_context(|| format!("listing {}", extract_root.display()))? - .collect::, _>>() - .with_context(|| format!("reading entries of {}", extract_root.display()))?; - entries.sort_by_key(|e| e.file_name()); - - let mut dirs = entries - .into_iter() - .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) - .map(|e| e.path()) - .collect::>(); - - match dirs.len() { - 1 => Ok(dirs.pop().expect("single dir")), - 0 => Err(anyhow!( - "expected one top-level folder under {}, found none", - extract_root.display() - )), - n => Err(anyhow!( - "expected one top-level folder under {}, found {n}", - extract_root.display() - )), - } -} - -pub async fn atomic_install(staged: &Path, final_dest: &Path) -> Result { - let staged = staged.to_path_buf(); - let final_dest = final_dest.to_path_buf(); - - tokio::task::spawn_blocking(move || -> Result { - if let Some(parent) = final_dest.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("creating parent {}", parent.display()))?; - } - - let backup = if final_dest.exists() { - let candidate = final_dest.with_extension(format!("old-{}", std::process::id())); - fs::rename(&final_dest, &candidate).with_context(|| { - format!( - "moving existing install {} aside to {}", - final_dest.display(), - candidate.display() - ) - })?; - Some(candidate) - } else { - None - }; - - if let Err(err) = fs::rename(&staged, &final_dest).with_context(|| { - format!( - "renaming staged {} -> {}", - staged.display(), - final_dest.display() - ) - }) { - if let Some(backup_path) = backup.as_ref() { - if let Err(restore_err) = fs::rename(backup_path, &final_dest) { - return Err(anyhow!( - "{err}; rollback from {} to {} also failed: {restore_err}", - backup_path.display(), - final_dest.display() - )); - } - } - return Err(err); - } - - if let Some(backup_path) = backup { - let _ = fs::remove_dir_all(backup_path); - } - - Ok(final_dest) - }) - .await - .context("spawn_blocking join failure during atomic install")? -} diff --git a/src/openhuman/runtime/python/mod.rs b/src/openhuman/runtime/python/mod.rs index 8171bc86a6..70fe16c24f 100644 --- a/src/openhuman/runtime/python/mod.rs +++ b/src/openhuman/runtime/python/mod.rs @@ -1,18 +1,16 @@ //! Managed Python runtime for Python-backed integrations. //! -//! The immediate use case is stdio MCP servers implemented in Python. This -//! module owns interpreter discovery and process-launch primitives so callers -//! do not need to care whether Python came from the host or a future managed -//! distribution. +//! [`bootstrap`] is the client for the `tinyruntime` module: it asks for an +//! interpreter and adapts the answer. Discovery, selection, download, and +//! install all live in the module now. +//! +//! [`process`] stays here because it is not runtime management. It launches the +//! long-lived stdio children this core owns — the runtime Python server, and the +//! stdio MCP servers — which outlive a single job and speak their own protocols. +//! The module resolves the interpreter; this core decides what to run with it. pub mod bootstrap; -pub mod downloader; -pub mod extractor; pub mod process; -pub mod resolver; pub use bootstrap::{PythonBootstrap, PythonSource, ResolvedPython}; -pub use downloader::{fetch_release_metadata, select_distribution, PythonDistribution}; -pub use extractor::{atomic_install, extract_distribution}; pub use process::PythonLaunchSpec; -pub use resolver::{detect_system_python, parse_python_version, PythonVersion, SystemPython}; diff --git a/src/openhuman/runtime/python/resolver.rs b/src/openhuman/runtime/python/resolver.rs deleted file mode 100644 index c11794a1a2..0000000000 --- a/src/openhuman/runtime/python/resolver.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! System Python resolver. -//! -//! Walks the configured command candidates / `PATH`, probes `--version`, and -//! returns a [`SystemPython`] when the interpreter satisfies the configured -//! minimum version floor. - -use std::ffi::OsString; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::Duration; - -/// Parsed Python semantic version. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub struct PythonVersion { - pub major: u32, - pub minor: u32, - pub patch: u32, -} - -impl PythonVersion { - pub fn display(self) -> String { - format!("{}.{}.{}", self.major, self.minor, self.patch) - } -} - -/// A usable Python interpreter discovered on the host. -#[derive(Debug, Clone)] -pub struct SystemPython { - /// Absolute path to the executable. - pub path: PathBuf, - /// Parsed semantic version. - pub version_info: PythonVersion, - /// Normalized `major.minor.patch` string. - pub version: String, -} - -/// Parse a version line like `Python 3.12.4` or `3.12.4`. -pub fn parse_python_version(raw: &str) -> Option { - let trimmed = raw.trim(); - let stripped = trimmed.strip_prefix("Python ").unwrap_or(trimmed); - let mut parts = stripped.split('.'); - let major = parts.next()?.parse::().ok()?; - let minor = parts.next()?.parse::().ok()?; - let patch = parts - .next() - .and_then(|segment| { - let digits = segment - .chars() - .take_while(|c| c.is_ascii_digit()) - .collect::(); - if digits.is_empty() { - None - } else { - digits.parse::().ok() - } - }) - .unwrap_or(0); - Some(PythonVersion { - major, - minor, - patch, - }) -} - -/// Probe the host for a Python interpreter satisfying `minimum_version`. -/// -/// Candidate order: -/// 1. `preferred_command` when supplied -/// 2. `python3.12` -/// 3. `python3` -/// 4. `python` -pub fn detect_system_python( - minimum_version: &str, - preferred_command: Option<&str>, -) -> Option { - detect_system_python_in_path( - minimum_version, - preferred_command, - std::env::var_os("PATH").as_ref(), - ) -} - -fn detect_system_python_in_path( - minimum_version: &str, - preferred_command: Option<&str>, - path_var: Option<&OsString>, -) -> Option { - let Some(minimum) = parse_python_version(minimum_version) else { - tracing::warn!( - minimum_version, - "[runtime_python::resolver] invalid minimum_version, skipping system-python probe" - ); - return None; - }; - - for candidate in candidate_commands(preferred_command, minimum) { - let Some(path) = resolve_candidate(&candidate, path_var) else { - tracing::debug!(candidate, "[runtime_python::resolver] candidate not found"); - continue; - }; - - tracing::debug!( - candidate, - path = %path.display(), - minimum_version = %minimum.display(), - "[runtime_python::resolver] probing python candidate" - ); - - let Some(raw_version) = probe_python_version(&path) else { - tracing::warn!( - candidate, - path = %path.display(), - "[runtime_python::resolver] `python --version` failed; skipping candidate" - ); - continue; - }; - - let Some(version_info) = parse_python_version(&raw_version) else { - tracing::warn!( - candidate, - path = %path.display(), - raw_version = %raw_version, - "[runtime_python::resolver] could not parse python version output" - ); - continue; - }; - - if version_info < minimum { - tracing::info!( - candidate, - path = %path.display(), - found = %version_info.display(), - minimum = %minimum.display(), - "[runtime_python::resolver] python candidate below minimum version" - ); - continue; - } - - let normalized = version_info.display(); - tracing::info!( - candidate, - path = %path.display(), - version = %normalized, - "[runtime_python::resolver] reusing compatible system python" - ); - return Some(SystemPython { - path, - version_info, - version: normalized, - }); - } - - None -} - -fn candidate_commands(preferred_command: Option<&str>, minimum: PythonVersion) -> Vec { - let mut candidates = Vec::new(); - if let Some(preferred) = preferred_command.map(str::trim).filter(|s| !s.is_empty()) { - candidates.push(preferred.to_string()); - } - let minimum_specific = format!("python{}.{}", minimum.major, minimum.minor); - for fallback in [minimum_specific.as_str(), "python3.12", "python3", "python"] { - if !candidates.iter().any(|existing| existing == fallback) { - candidates.push(fallback.to_string()); - } - } - candidates -} - -fn resolve_candidate(candidate: &str, path_var: Option<&OsString>) -> Option { - let path = Path::new(candidate); - if path.components().count() > 1 || path.is_absolute() { - return is_executable_candidate(path).then(|| path.to_path_buf()); - } - - let path_var = path_var?; - for dir in std::env::split_paths(path_var) { - let base = dir.join(candidate); - if is_executable_candidate(&base) { - return Some(base); - } - #[cfg(windows)] - { - let exe = dir.join(format!("{candidate}.exe")); - if is_executable_candidate(&exe) { - return Some(exe); - } - } - } - None -} - -#[cfg(unix)] -fn is_executable_candidate(path: &Path) -> bool { - use std::os::unix::fs::PermissionsExt; - std::fs::metadata(path) - .map(|meta| meta.is_file() && (meta.permissions().mode() & 0o111 != 0)) - .unwrap_or(false) -} - -#[cfg(not(unix))] -fn is_executable_candidate(path: &Path) -> bool { - path.is_file() -} - -fn probe_python_version(path: &Path) -> Option { - use std::io::Read; - use wait_timeout::ChildExt; - - let mut cmd = Command::new(path); - cmd.arg("--version") - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - #[cfg(windows)] - { - use std::os::windows::process::CommandExt; - cmd.creation_flags(0x0800_0000); // CREATE_NO_WINDOW - } - let mut child = cmd.spawn().ok()?; - let timeout = Duration::from_secs(5); - let status = match child.wait_timeout(timeout).ok()? { - Some(status) => status, - None => { - tracing::warn!( - path = %path.display(), - timeout_secs = 5, - "[runtime_python::resolver] ` --version` timed out; killing process" - ); - let _ = child.kill(); - let _ = child.wait(); - return None; - } - }; - - if !status.success() { - let mut stderr_buf = String::new(); - if let Some(mut s) = child.stderr.take() { - let _ = s.read_to_string(&mut stderr_buf); - } - tracing::debug!( - path = %path.display(), - status = ?status, - stderr = %stderr_buf, - "[runtime_python::resolver] ` --version` exited non-zero" - ); - return None; - } - - let mut stdout_buf = String::new(); - if let Some(mut s) = child.stdout.take() { - let _ = s.read_to_string(&mut stdout_buf); - } - let mut stderr_buf = String::new(); - if let Some(mut s) = child.stderr.take() { - let _ = s.read_to_string(&mut stderr_buf); - } - - let combined = if stdout_buf.trim().is_empty() { - stderr_buf.trim().to_string() - } else { - stdout_buf.trim().to_string() - }; - if combined.is_empty() { - None - } else { - Some(combined) - } -} - -pub(crate) fn probe_python_version_public(path: &Path) -> Option { - probe_python_version(path) -} - -#[cfg(test)] -#[path = "resolver_tests.rs"] -mod tests; diff --git a/src/openhuman/runtime/python/resolver_tests.rs b/src/openhuman/runtime/python/resolver_tests.rs deleted file mode 100644 index 624d668606..0000000000 --- a/src/openhuman/runtime/python/resolver_tests.rs +++ /dev/null @@ -1,96 +0,0 @@ -use super::*; - -#[test] -fn parses_standard_python_version() { - assert_eq!( - parse_python_version("Python 3.12.4"), - Some(PythonVersion { - major: 3, - minor: 12, - patch: 4 - }) - ); -} - -#[test] -fn parses_without_python_prefix() { - assert_eq!( - parse_python_version("3.12.0"), - Some(PythonVersion { - major: 3, - minor: 12, - patch: 0 - }) - ); -} - -#[test] -fn parses_patchless_version_as_zero() { - assert_eq!( - parse_python_version("Python 3.12"), - Some(PythonVersion { - major: 3, - minor: 12, - patch: 0 - }) - ); -} - -#[test] -fn rejects_invalid_versions() { - assert_eq!(parse_python_version("Python three.twelve"), None); - assert_eq!(parse_python_version(""), None); -} - -#[cfg(unix)] -#[test] -fn detects_preferred_python_from_custom_path() { - use std::fs; - use std::os::unix::fs::PermissionsExt; - - let dir = tempfile::tempdir().expect("tempdir"); - let script = dir.path().join("python3.12"); - fs::write(&script, "#!/bin/sh\necho 'Python 3.12.7'\n").expect("write script"); - let mut perms = fs::metadata(&script).expect("metadata").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script, perms).expect("chmod"); - - let path_var = OsString::from(dir.path().display().to_string()); - let found = detect_system_python_in_path("3.12.0", Some("python3.12"), Some(&path_var)) - .expect("python should resolve"); - - assert_eq!(found.version, "3.12.7"); - assert_eq!(found.path, script); -} - -#[cfg(unix)] -#[test] -fn rejects_python_below_minimum() { - use std::fs; - use std::os::unix::fs::PermissionsExt; - - let dir = tempfile::tempdir().expect("tempdir"); - let script = dir.path().join("python3"); - fs::write(&script, "#!/bin/sh\necho 'Python 3.11.9'\n").expect("write script"); - let mut perms = fs::metadata(&script).expect("metadata").permissions(); - perms.set_mode(0o755); - fs::set_permissions(&script, perms).expect("chmod"); - - let path_var = OsString::from(dir.path().display().to_string()); - let found = detect_system_python_in_path("3.12.0", None, Some(&path_var)); - assert!(found.is_none(), "3.11 must be rejected"); -} - -#[test] -fn candidate_commands_include_minimum_specific_binary() { - let candidates = candidate_commands( - None, - PythonVersion { - major: 3, - minor: 13, - patch: 0, - }, - ); - assert_eq!(candidates[0], "python3.13"); - assert!(candidates.iter().any(|candidate| candidate == "python3")); -} From 91883a1835d065e6fee8d793f4ce88c26f7a0807 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:10:10 +0300 Subject: [PATCH 15/55] refactor(runtime/pool): delegate pool implementation to tinyruntime module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pool implementation — warm interpreter children, the newline-delimited job protocol, backpressure, idle reaping, and recycle-after-N — has been moved into the `tinyruntime` module so that one implementation serves every language. This change removes the local pool, worker, protocol, environment, and harness files, and rewrites the module root to be a thin client that delegates to the module and maps its replies back onto the shapes the existing exec tools already handle. The three-way `PoolRunError` distinction (saturated, pre-dispatch, post-dispatch) is preserved locally because `node_exec` and `python_exec` match on it to decide between reporting a result, falling back to a per-call spawn, and refusing to retry. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/pool/env.rs | 126 ------- src/openhuman/runtime/pool/mod.rs | 142 +++++-- src/openhuman/runtime/pool/pool.rs | 435 ---------------------- src/openhuman/runtime/pool/pool_worker.js | 292 --------------- src/openhuman/runtime/pool/pool_worker.py | 212 ----------- src/openhuman/runtime/pool/protocol.rs | 125 ------- src/openhuman/runtime/pool/types.rs | 119 +++--- src/openhuman/runtime/pool/worker.rs | 378 ------------------- 8 files changed, 176 insertions(+), 1653 deletions(-) delete mode 100644 src/openhuman/runtime/pool/env.rs delete mode 100644 src/openhuman/runtime/pool/pool.rs delete mode 100644 src/openhuman/runtime/pool/pool_worker.js delete mode 100644 src/openhuman/runtime/pool/pool_worker.py delete mode 100644 src/openhuman/runtime/pool/protocol.rs delete mode 100644 src/openhuman/runtime/pool/worker.rs diff --git a/src/openhuman/runtime/pool/env.rs b/src/openhuman/runtime/pool/env.rs deleted file mode 100644 index 4c75f9188f..0000000000 --- a/src/openhuman/runtime/pool/env.rs +++ /dev/null @@ -1,126 +0,0 @@ -//! Worker environment + harness-script materialisation helpers. -//! -//! Split out of `mod.rs` so the module root stays export-focused. Owns the -//! allow-listed child environment (`base_env`) and the once-per-process harness -//! write (`ensure_worker_script`). - -use std::path::{Path, PathBuf}; - -use anyhow::{Context, Result}; -use tokio::sync::OnceCell; - -/// Env vars forwarded (allow-listed) into pooled workers. Mirrors the -/// `node_exec` / shell hygiene: secrets never leak into a worker's environment; -/// `PATH` is rebuilt separately with the managed interpreter's bin dir first. -const SAFE_ENV_VARS: &[&str] = &[ - "HOME", - "TERM", - "LANG", - "LC_ALL", - "LC_CTYPE", - "USER", - "SHELL", - "TMPDIR", - // Windows process creation + child command lookup after env_clear(). - "SystemRoot", - "WINDIR", - "COMSPEC", - "PATHEXT", - "TEMP", - "TMP", - "USERPROFILE", - "APPDATA", - "LOCALAPPDATA", - "ProgramFiles", - "ProgramFiles(x86)", - "ProgramW6432", -]; - -/// Build the allow-listed environment for a worker, with `bin_dir` prepended to -/// `PATH` so the child resolves the managed interpreter (and its tools). -pub(crate) fn base_env(bin_dir: &Path) -> Vec<(String, String)> { - let mut env: Vec<(String, String)> = Vec::new(); - - let host_path = std::env::var("PATH").unwrap_or_default(); - let sep = if cfg!(windows) { ";" } else { ":" }; - let path = if host_path.is_empty() { - bin_dir.to_string_lossy().into_owned() - } else { - format!("{}{}{}", bin_dir.display(), sep, host_path) - }; - env.push(("PATH".to_string(), path)); - - for var in SAFE_ENV_VARS { - if let Ok(val) = std::env::var(var) { - env.push(((*var).to_string(), val)); - } - } - env -} - -/// Materialise a bundled harness script into a stable per-workspace cache path -/// and return it. -async fn write_worker_script( - workspace_dir: &Path, - filename: &str, - contents: &str, -) -> Result { - let root = workspace_dir.join("runtime_pool"); - tracing::debug!(dir = %root.display(), filename, "[runtime_pool] writing worker harness"); - tokio::fs::create_dir_all(&root) - .await - .with_context(|| format!("creating runtime_pool cache {}", root.display()))?; - let path = root.join(filename); - tokio::fs::write(&path, contents) - .await - .with_context(|| format!("writing worker script {}", path.display()))?; - tracing::debug!(path = %path.display(), bytes = contents.len(), "[runtime_pool] worker harness ready"); - Ok(path) -} - -/// Return the harness script path, writing it **once per process** (a hot-path -/// `node_exec`/`python_exec` must not touch disk on every call — the point of -/// #5106 is to *reduce* per-run cost). The script is written on the first inline -/// exec and cached; a core upgrade is a fresh process, so it re-materialises -/// then, keeping the shipped harness current. -pub(crate) async fn ensure_worker_script( - cell: &'static OnceCell, - workspace_dir: &Path, - filename: &str, - contents: &str, -) -> Result { - Ok(cell - .get_or_try_init(|| write_worker_script(workspace_dir, filename, contents)) - .await? - .clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn base_env_prepends_bin_dir_to_path() { - let env = base_env(Path::new("/managed/bin")); - let path = env - .iter() - .find(|(k, _)| k == "PATH") - .map(|(_, v)| v.clone()) - .expect("PATH present"); - assert!( - path.starts_with("/managed/bin"), - "bin dir must be first on PATH; got {path}" - ); - } - - #[tokio::test] - async fn write_worker_script_roundtrips() { - let tmp = std::env::temp_dir().join(format!("rt-pool-test-{}", std::process::id())); - let path = write_worker_script(&tmp, "probe.js", "console.log('hi')") - .await - .expect("script written"); - let read = tokio::fs::read_to_string(&path).await.unwrap(); - assert_eq!(read, "console.log('hi')"); - let _ = tokio::fs::remove_dir_all(&tmp).await; - } -} diff --git a/src/openhuman/runtime/pool/mod.rs b/src/openhuman/runtime/pool/mod.rs index 141596c135..d6f41b94c5 100644 --- a/src/openhuman/runtime/pool/mod.rs +++ b/src/openhuman/runtime/pool/mod.rs @@ -1,43 +1,121 @@ -//! Shared, bounded pools of long-lived `node` / `python` worker processes that -//! execute inline code jobs for skill runs and the `node_exec` agent tool — -//! instead of forking one interpreter child per execution (issue #5106). +//! Pooled execution of inline code, delegated to the `tinyruntime` module. //! -//! ## Why +//! The pool itself — warm interpreter children, the newline-delimited job +//! protocol, backpressure, idle reaping, recycle-after-N — moved into that +//! module, where one implementation serves every language. What is left here is +//! the client: whether this host wants pooling for a language, and the mapping +//! from a module reply back onto the shapes the exec tools already handle. //! -//! A single JS skill step spawns a `node` child at ~72–75 MB RSS. At the -//! opencompany target (100–1000 live agents in 2 GB / 2 vCPU) those per-run -//! interpreter children are the biggest budget breaker. Sharing a small bounded -//! pool of warm workers turns *K concurrent skill runs → K interpreters* into -//! *K concurrent skill runs → ~one pooled worker*, trading a little latency -//! (work beyond the pool size queues) for a large, flat memory floor. +//! # Why the local types survived //! -//! ## Shape +//! [`PoolExecOutcome`] and [`PoolRunError`] are what `node_exec` and +//! `python_exec` match on to decide between reporting a result, falling back to +//! a per-call spawn, and refusing to retry. Keeping them meant the migration did +//! not have to rewrite either tool's dispatch logic — and the three-way +//! distinction they encode is the one that keeps a job from running twice. //! -//! * [`worker`] — one warm interpreter child speaking newline-delimited JSON. -//! * [`pool`] — the bounded [`LangPool`](pool::LangPool): semaphore-gated -//! concurrency, queue backpressure, idle-TTL reaping, recycle-after-N-jobs, -//! plus the process-global registry keyed per language. -//! * [`node`] / [`python`] — language backends that resolve the interpreter, -//! materialise the harness script, and submit inline jobs. -//! * [`env`] — allow-listed worker environment + once-per-process harness write. +//! # The fallback is still real //! -//! The whole subsystem is an **optimisation seam**: `runtime_pool.enabled = -//! false` (or a per-language flag) reverts callers to their legacy per-call -//! spawn with no behavioural change. +//! A pre-dispatch failure sends the caller to its legacy per-call spawn, which +//! still exists and still works: the tools hold a resolved interpreter path from +//! the module and can run it directly. Pooling is an optimisation seam, exactly +//! as it was — `runtime_pool.enabled = false` reverts every caller with no +//! behavioural change. -pub mod env; pub mod node; -// `module_inception` is a byproduct of the domain-family reorg: the parent was -// renamed from `runtime_pool` to `runtime/pool`, which shortened it to match this -// long-standing inner module. Renaming the inner module would be a real rename -// on top of a pure move, so it is allowed here and left as follow-up. -#[allow(clippy::module_inception)] -pub mod pool; -pub mod protocol; pub mod python; pub mod types; -pub mod worker; -pub(crate) use env::{base_env, ensure_worker_script}; -pub use pool::{all_stats, LangPool, PoolRunError, PoolStats}; +use tinyruntime_bus::Language; + +use crate::openhuman::config::Config; +use crate::openhuman::modules::runtime::{self, RuntimeCallError}; + pub use types::{PoolExecOutcome, PoolLang, PoolSettings}; + +/// Why a pooled run failed, classified so callers know what is safe to do next. +#[derive(Debug)] +pub enum PoolRunError { + /// The pool was at capacity and shed the job rather than buffering it. + /// + /// Callers must **not** fall back to a per-call spawn: that reintroduces the + /// very resident memory the pool exists to cap. Surface a busy error or + /// retry later. + Saturated, + /// The job never reached a worker, so it never ran. A retry or a legacy + /// per-call spawn is safe. + PreDispatch(anyhow::Error), + /// The job reached a worker and may have executed. Terminal — re-running it + /// could duplicate whatever it already did. + PostDispatch(anyhow::Error), +} + +impl std::fmt::Display for PoolRunError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Saturated => write!(f, "runtime pool at capacity"), + Self::PreDispatch(error) => write!(f, "pre-dispatch pool failure: {error:#}"), + Self::PostDispatch(error) => write!(f, "post-dispatch pool failure: {error:#}"), + } + } +} + +/// Run one inline job on `language`'s pool. +/// +/// Shared by the two language backends, which differ only in which language they +/// name — every other decision belongs to the module. +async fn run_inline( + config: &Config, + language: &Language, + code: String, + cwd: Option, + timeout: Option, +) -> Result { + let cwd = cwd.map(|path| path.to_string_lossy().into_owned()); + + match runtime::execute(config, language, code, cwd, timeout).await { + Ok(response) => Ok(PoolExecOutcome::from_module(&response)), + Err(error) => Err(classify(&error)), + } +} + +/// Map a module failure onto the three-way distinction callers act on. +/// +/// The saturation and post-dispatch readings come from the module's own error +/// text, which is its contract with a host that renders it. Anything else is +/// pre-dispatch: the conservative reading for a *retryable* classification would +/// be wrong in the other direction — assuming a job ran when it did not merely +/// costs a fallback spawn, while assuming it did not when it did would run +/// someone's code twice. +fn classify(error: &RuntimeCallError) -> PoolRunError { + let message = error.to_string(); + if message.contains("pool is at capacity") { + return PoolRunError::Saturated; + } + if message.contains("failed after dispatch") { + return PoolRunError::PostDispatch(anyhow::anyhow!("{message}")); + } + PoolRunError::PreDispatch(anyhow::anyhow!("{message}")) +} + +/// Every live pool's counters, as the module reports them. +/// +/// Returns an empty list when the module is not loaded or has no pool yet, which +/// is the same thing a status surface wants to render: nothing running. +pub async fn all_stats(config: &Config) -> Vec<(PoolLang, tinyruntime_bus::PoolStats)> { + match runtime::pool_stats(config).await { + Ok(response) => response + .pools + .into_iter() + .filter_map(|stats| PoolLang::from_language(&stats.language).map(|lang| (lang, stats))) + .collect(), + Err(error) => { + tracing::debug!("[runtime::pool] pool stats are unavailable: {error}"); + Vec::new() + } + } +} + +#[cfg(test)] +#[path = "pool_tests.rs"] +mod tests; diff --git a/src/openhuman/runtime/pool/pool.rs b/src/openhuman/runtime/pool/pool.rs deleted file mode 100644 index 9efd7eb899..0000000000 --- a/src/openhuman/runtime/pool/pool.rs +++ /dev/null @@ -1,435 +0,0 @@ -//! The bounded per-language worker pool and its process-global registry. -//! -//! One [`LangPool`] owns up to `max_workers` warm [`PoolWorker`]s for a single -//! language. Concurrency is bounded by a semaphore; submissions beyond the pool -//! size **queue** on that semaphore (the intended backpressure), and -//! submissions beyond `max_workers + max_queue_depth` in flight are rejected so -//! memory can't grow without bound. Idle workers are reaped after a TTL; busy -//! workers are recycled after N jobs. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, OnceLock, Weak}; -use std::time::{Duration, Instant}; - -use anyhow::Result; -use tokio::sync::{Mutex, Semaphore}; - -use super::protocol::{PoolJobRequest, PoolJobResponse}; -use super::types::{PoolExecOutcome, PoolLang, PoolSettings}; -use super::worker::{PoolWorker, WorkerLaunch}; - -/// Why a pooled run failed, classified so callers know whether a retry or a -/// legacy per-call-spawn fallback is safe. -#[derive(Debug)] -pub enum PoolRunError { - /// In-flight work exceeded `max_workers + max_queue_depth`; the pool shed - /// load rather than buffering unbounded. Callers must **not** fall back to a - /// per-call spawn — that reintroduces the very RSS the pool caps (#5106) — - /// but surface a busy error or retry later. - Saturated, - /// Failure before the job reached a worker (serialise / spawn / write). The - /// job never ran, so a retry or a legacy-spawn fallback is safe. - PreDispatch(anyhow::Error), - /// Failure after the job was dispatched (it may have executed). Terminal — - /// the caller must not re-run it (would duplicate side effects). - PostDispatch(anyhow::Error), -} - -impl std::fmt::Display for PoolRunError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PoolRunError::Saturated => write!(f, "runtime pool at capacity"), - PoolRunError::PreDispatch(e) => write!(f, "pre-dispatch pool failure: {e:#}"), - PoolRunError::PostDispatch(e) => write!(f, "post-dispatch pool failure: {e:#}"), - } - } -} - -/// Extra grace added to a job's soft deadline before the Rust side treats the -/// worker as wedged and kills it. The worker should always self-abort first. -const HARD_TIMEOUT_GRACE: Duration = Duration::from_secs(10); - -/// Lower bound on how often the idle reaper wakes, so a tiny TTL doesn't busy-loop. -const MIN_REAP_INTERVAL: Duration = Duration::from_secs(5); - -/// Lightweight, cloneable snapshot of a pool's counters (for status/tests). -#[derive(Debug, Clone, Default)] -pub struct PoolStats { - pub jobs_total: u64, - pub worker_spawns: u64, - pub rejected_saturated: u64, - pub idle_workers: usize, - pub max_workers: usize, -} - -pub struct LangPool { - launch: WorkerLaunch, - settings: PoolSettings, - /// Permits == max_workers. Acquiring one is the queue/backpressure gate. - permits: Arc, - /// Warm, idle workers available for reuse. - idle: Mutex>, - /// Jobs currently in flight or waiting for a permit (for saturation guard). - inflight: AtomicUsize, - job_seq: AtomicU64, - jobs_total: AtomicU64, - worker_spawns: AtomicU64, - rejected_saturated: AtomicU64, -} - -impl LangPool { - /// Build a pool and start its idle reaper. The reaper holds only a `Weak` - /// ref, so the pool is still dropped normally when the registry evicts it. - pub fn start(launch: WorkerLaunch, settings: PoolSettings) -> Arc { - let permits = Arc::new(Semaphore::new(settings.max_workers)); - let pool = Arc::new(Self { - launch, - settings, - permits, - idle: Mutex::new(Vec::new()), - inflight: AtomicUsize::new(0), - job_seq: AtomicU64::new(0), - jobs_total: AtomicU64::new(0), - worker_spawns: AtomicU64::new(0), - rejected_saturated: AtomicU64::new(0), - }); - if let Some(ttl) = pool.settings.idle_ttl { - spawn_reaper(Arc::downgrade(&pool), ttl); - } - pool - } - - pub fn lang(&self) -> PoolLang { - self.launch.lang - } - - pub async fn stats(&self) -> PoolStats { - PoolStats { - jobs_total: self.jobs_total.load(Ordering::Relaxed), - worker_spawns: self.worker_spawns.load(Ordering::Relaxed), - rejected_saturated: self.rejected_saturated.load(Ordering::Relaxed), - idle_workers: self.idle.lock().await.len(), - max_workers: self.settings.max_workers, - } - } - - /// Run one inline job, blocking (asynchronously) until a worker is free. - pub async fn run_inline( - &self, - code: String, - cwd: Option, - timeout: Option, - ) -> Result { - // Saturation guard: bound total in-flight (running + queued) work so a - // stampede queues up to a point, then sheds load instead of buffering - // unbounded. Capacity = worker slots + allowed queue depth. - let capacity = self.settings.max_workers + self.settings.max_queue_depth; - let inflight_now = self.inflight.fetch_add(1, Ordering::AcqRel) + 1; - if inflight_now > capacity { - self.inflight.fetch_sub(1, Ordering::AcqRel); - self.rejected_saturated.fetch_add(1, Ordering::Relaxed); - tracing::warn!( - lang = self.launch.lang.id(), - inflight = inflight_now, - capacity, - "[runtime_pool] saturated; shedding load (no spawn fallback)" - ); - return Err(PoolRunError::Saturated); - } - // Ensure the in-flight counter is released on every exit path below. - let _inflight_guard = InflightGuard(&self.inflight); - - let wait_start = Instant::now(); - let _permit = self - .permits - .acquire() - .await - .expect("runtime pool semaphore never closed"); - let queue_wait = wait_start.elapsed(); - - let id = self.job_seq.fetch_add(1, Ordering::Relaxed).to_string(); - let req = PoolJobRequest { - id, - kind: "inline".to_string(), - code: Some(code), - cwd, - timeout_ms: timeout.map(|t| t.as_millis() as u64), - }; - let hard_timeout = timeout.map(|t| t + HARD_TIMEOUT_GRACE); - - let job_start = Instant::now(); - let (response, worker) = self.submit_with_retry(&req, hard_timeout).await?; - let elapsed = job_start.elapsed(); - - self.jobs_total.fetch_add(1, Ordering::Relaxed); - - // Recycle after N jobs, otherwise return the warm worker to the pool. - if worker.should_recycle(self.settings.recycle_after_jobs) { - tracing::debug!( - lang = self.launch.lang.id(), - jobs = worker.jobs_done(), - "[runtime_pool] recycling worker after job budget" - ); - worker.shutdown(); - } else { - self.idle.lock().await.push(worker); - } - - if let Some(err) = response.error { - // The worker replied with a harness-level error: the job was - // dispatched (and may have run), so this is terminal. - return Err(PoolRunError::PostDispatch(anyhow::anyhow!( - "{} worker error: {err}", - self.launch.lang.id() - ))); - } - Ok(PoolExecOutcome { - stdout: response.stdout, - stderr: response.stderr, - exit_code: response.exit_code, - timed_out: response.timed_out, - elapsed, - queue_wait, - }) - } - - /// Submit on a warm-or-fresh worker. A **pre-dispatch** failure (e.g. a - /// reused idle worker that died on write) respawns once — the job never ran, - /// so that is safe. A **post-dispatch** failure is terminal: the job may have - /// executed, so it is never re-run (no duplicate side effects). Returns the - /// surviving worker so the caller can recycle or re-pool it. - async fn submit_with_retry( - &self, - req: &PoolJobRequest, - hard_timeout: Option, - ) -> Result<(PoolJobResponse, PoolWorker), PoolRunError> { - let mut worker = self - .take_or_spawn() - .await - .map_err(PoolRunError::PreDispatch)?; - match worker.submit(req, hard_timeout).await { - Ok(resp) => Ok((resp, worker)), - Err(e) if !e.dispatched => { - tracing::warn!( - lang = self.launch.lang.id(), - "[runtime_pool] pre-dispatch submit failure ({e}); respawning once" - ); - worker.shutdown(); - let mut fresh = self - .spawn_worker() - .await - .map_err(PoolRunError::PreDispatch)?; - match fresh.submit(req, hard_timeout).await { - Ok(resp) => Ok((resp, fresh)), - Err(e2) if !e2.dispatched => Err(PoolRunError::PreDispatch(e2.err)), - Err(e2) => Err(PoolRunError::PostDispatch(e2.err)), - } - } - Err(e) => { - tracing::warn!( - lang = self.launch.lang.id(), - "[runtime_pool] post-dispatch submit failure ({e}); terminal, not retrying" - ); - worker.shutdown(); - Err(PoolRunError::PostDispatch(e.err)) - } - } - } - - /// Pop a still-fresh idle worker, or spawn a new one. - async fn take_or_spawn(&self) -> Result { - { - let mut idle = self.idle.lock().await; - while let Some(worker) = idle.pop() { - if let Some(ttl) = self.settings.idle_ttl { - if worker.idle_expired(ttl) { - worker.shutdown(); - continue; - } - } - return Ok(worker); - } - } - self.spawn_worker().await - } - - async fn spawn_worker(&self) -> Result { - self.worker_spawns.fetch_add(1, Ordering::Relaxed); - PoolWorker::spawn(&self.launch).await - } - - /// Drop workers idle beyond the TTL. Called by the background reaper. - async fn reap_idle(&self) { - let Some(ttl) = self.settings.idle_ttl else { - return; - }; - let mut idle = self.idle.lock().await; - let before = idle.len(); - let mut kept = Vec::with_capacity(before); - for worker in idle.drain(..) { - if worker.idle_expired(ttl) { - worker.shutdown(); - } else { - kept.push(worker); - } - } - let reaped = before - kept.len(); - *idle = kept; - if reaped > 0 { - tracing::debug!( - lang = self.launch.lang.id(), - reaped, - "[runtime_pool] idle reaper retired workers" - ); - } - } -} - -/// Decrements the in-flight counter on drop so every early return is covered. -struct InflightGuard<'a>(&'a AtomicUsize); -impl Drop for InflightGuard<'_> { - fn drop(&mut self) { - self.0.fetch_sub(1, Ordering::AcqRel); - } -} - -fn spawn_reaper(pool: Weak, ttl: Duration) { - let interval = ttl.max(MIN_REAP_INTERVAL); - tokio::spawn(async move { - loop { - tokio::time::sleep(interval).await; - match pool.upgrade() { - Some(pool) => pool.reap_idle().await, - None => break, // pool dropped — stop reaping - } - } - }); -} - -// --------------------------------------------------------------------------- -// Process-global registry -// --------------------------------------------------------------------------- - -struct CachedPool { - key: String, - pool: Arc, -} - -static REGISTRY: OnceLock>> = OnceLock::new(); - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum PoolLangKey { - Node, - Python, -} - -impl From for PoolLangKey { - fn from(lang: PoolLang) -> Self { - match lang { - PoolLang::Node => PoolLangKey::Node, - PoolLang::Python => PoolLangKey::Python, - } - } -} - -fn registry() -> &'static Mutex> { - REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) -} - -/// Fingerprint that decides whether a cached pool can be reused. Any change to -/// the interpreter path, args, or tuning rebuilds the pool. -fn launch_key(launch: &WorkerLaunch, settings: &PoolSettings) -> String { - format!( - "{}|{}|{:?}|isolated={}|w={}|ttl={:?}|recycle={}|q={}", - launch.lang.id(), - launch.bin.display(), - launch.args, - launch.isolated_protocol, - settings.max_workers, - settings.idle_ttl, - settings.recycle_after_jobs, - settings.max_queue_depth, - ) -} - -/// Get (or build) the process-global pool for a language, keyed by its launch -/// fingerprint. A config or interpreter change transparently rebuilds it. -pub async fn ensure_pool(launch: WorkerLaunch, settings: PoolSettings) -> Arc { - let key = launch_key(&launch, &settings); - let lang_key = PoolLangKey::from(launch.lang); - let mut reg = registry().lock().await; - if let Some(cached) = reg.get(&lang_key) { - if cached.key == key { - return cached.pool.clone(); - } - tracing::info!( - lang = launch.lang.id(), - "[runtime_pool] launch spec changed; rebuilding pool" - ); - } - let pool = LangPool::start(launch, settings); - reg.insert( - lang_key, - CachedPool { - key, - pool: pool.clone(), - }, - ); - pool -} - -/// Snapshot every live pool's stats (for a status surface / debugging). -pub async fn all_stats() -> Vec<(PoolLang, PoolStats)> { - let reg = registry().lock().await; - let mut out = Vec::new(); - for cached in reg.values() { - out.push((cached.pool.lang(), cached.pool.stats().await)); - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn launch_key_changes_with_tuning() { - let launch = WorkerLaunch { - lang: PoolLang::Node, - bin: "/usr/bin/node".into(), - args: vec!["worker.js".into()], - env: vec![], - isolated_protocol: true, - }; - let a = PoolSettings { - max_workers: 2, - idle_ttl: Some(Duration::from_secs(60)), - recycle_after_jobs: 100, - max_queue_depth: 256, - }; - let mut b = a.clone(); - b.max_workers = 4; - assert_ne!(launch_key(&launch, &a), launch_key(&launch, &b)); - assert_eq!(launch_key(&launch, &a), launch_key(&launch, &a)); - } - - #[test] - fn pool_run_error_display_is_classified() { - // The three arms drive distinct caller behaviour (retry / fall back / - // give up), so their rendered messages must stay distinguishable. - assert_eq!( - PoolRunError::Saturated.to_string(), - "runtime pool at capacity" - ); - let pre = PoolRunError::PreDispatch(anyhow::anyhow!("spawn failed")).to_string(); - assert!(pre.starts_with("pre-dispatch pool failure:"), "got {pre}"); - assert!(pre.contains("spawn failed")); - let post = PoolRunError::PostDispatch(anyhow::anyhow!("read wedged")).to_string(); - assert!( - post.starts_with("post-dispatch pool failure:"), - "got {post}" - ); - assert!(post.contains("read wedged")); - } -} diff --git a/src/openhuman/runtime/pool/pool_worker.js b/src/openhuman/runtime/pool/pool_worker.js deleted file mode 100644 index e6b674dab1..0000000000 --- a/src/openhuman/runtime/pool/pool_worker.js +++ /dev/null @@ -1,292 +0,0 @@ -// OpenHuman runtime-pool Node worker harness (issue #5106). -// -// A single long-lived `node` process that executes inline JavaScript jobs for -// many skill runs / node_exec calls, so the fleet pays one warm interpreter -// instead of one child per run. -// -// Protocol (newline-delimited JSON over an authenticated loopback socket, -// see runtime_pool/protocol.rs): -// 1. Print exactly one ready line: {"ready":true,"protocol":1,"lang":"node"} -// 2. For each request line {id,kind:"inline",code,cwd,timeout_ms} reply with -// {id,ok,stdout,stderr,exit_code,timed_out,elapsed_ms,error}. -// -// Each job runs in its own `worker_thread` for isolation (fresh module graph + -// globals per run) and safe termination (a runaway or process.exit()-y job is -// killed with worker.terminate() without taking down this host process). The -// job's stdout/stderr are isolated pipes (stdout:true/stderr:true). Protocol -// replies use a separate authenticated loopback socket so fd-level writes -// (`fs.writeSync`, inherited child stdio) cannot forge or corrupt frames. - -'use strict'; - -const { Worker, isMainThread, parentPort, workerData } = require('worker_threads'); - -const PROTOCOL_VERSION = 1; - -// --------------------------------------------------------------------------- -// Worker-thread mode: execute one job's code, then exit (flushing its pipes). -// --------------------------------------------------------------------------- -if (!isMainThread) { - const path = require('path'); - const vm = require('vm'); - const { createRequire } = require('module'); - const { pathToFileURL } = require('url'); - - async function runUserCode(code, cwd) { - // NOTE: do NOT `process.chdir()` here — it throws ERR_WORKER_UNSUPPORTED_ - // OPERATION inside a worker thread. The host chdirs before spawning this - // worker (jobs serialize per worker process), so `process.cwd()` is already - // the job's directory; `cwd` roots require/__dirname and the import base. - const dir = cwd || process.cwd(); - const filename = path.join(dir, 'inline.js'); - const req = createRequire(filename); - const base = pathToFileURL(filename).href; - // Use the ESM resolver (not createRequire.resolve) for bare dynamic - // imports so import-only package exports retain `node -e` semantics. - const { default: resolveEsm } = await import( - 'data:text/javascript,export default (specifier, parent) => import.meta.resolve(specifier, parent)' - ); - const importFromJob = (specifier, _referrer, importAttributes) => { - const options = - importAttributes && Object.keys(importAttributes).length > 0 - ? { with: importAttributes } - : undefined; - if ( - specifier.startsWith('.') || - specifier.startsWith('/') || - specifier.startsWith('file:') || - specifier.startsWith('data:') - ) { - return import(new URL(specifier, base).href, options); - } - return import(resolveEsm(specifier, base), options); - }; - // Mimic `node -e`: CommonJS-ish sloppy scope with require/__dirname, wrapped - // in an async IIFE so top-level `await` works. `vm.compileFunction` (over a - // bare `new Function`) lets us root dynamic `import()` at the job cwd via - // `importModuleDynamically`, so `await import('./rel.mjs')` resolves like - // `node -e` instead of relative to this harness file. Needs - // `--experimental-vm-modules` (passed on the worker launch). - const fn = vm.compileFunction( - 'return (async () => {\n' + code + '\n})();', - ['require', '__filename', '__dirname', 'module', 'exports'], - { - filename, - importModuleDynamically: importFromJob, - } - ); - const mod = { exports: {} }; - await fn(req, filename, dir, mod, mod.exports); - } - - const code = (workerData && workerData.code) || ''; - const cwd = (workerData && workerData.cwd) || null; - runUserCode(code, cwd).then( - () => { - // Resolve → let the thread exit naturally once its loop drains, which - // flushes the stdout/stderr pipes before the 'exit' event fires. - }, - (err) => { - const msg = err && err.stack ? err.stack : String(err); - process.stderr.write(msg + '\n'); - process.exitCode = 1; - } - ); - return; -} - -// --------------------------------------------------------------------------- -// Main (host) mode: read jobs, run each in a worker thread, reply per job. -// --------------------------------------------------------------------------- - -function collect(stream) { - return new Promise((resolve) => { - let buf = ''; - stream.setEncoding('utf8'); - stream.on('data', (d) => { - buf += d; - }); - const done = () => resolve(buf); - stream.on('end', done); - stream.on('close', done); - stream.on('error', done); - }); -} - -function runJob(job) { - return new Promise((resolve) => { - const start = Date.now(); - // Set the job's working directory on the HOST before spawning the worker: - // a worker thread inherits the parent's cwd at creation and cannot chdir - // itself. The worker captures cwd synchronously at construction, so we - // restore the host's prior cwd immediately after — otherwise a later job - // without `cwd` (or whose chdir failed) would silently inherit this job's - // directory instead of the worker's original one. - const priorCwd = process.cwd(); - if (job.cwd) { - try { - process.chdir(job.cwd); - } catch (e) { - resolve({ - id: job.id, - ok: false, - stdout: '', - stderr: '', - exit_code: null, - timed_out: false, - elapsed_ms: Date.now() - start, - error: 'failed to set worker cwd: ' + (e && e.stack ? e.stack : String(e)), - }); - return; - } - } - let worker; - try { - worker = new Worker(__filename, { - workerData: { id: job.id, code: job.code || '', cwd: job.cwd || null }, - stdout: true, - stderr: true, - // Propagate host node flags (e.g. --experimental-vm-modules) so the - // worker's vm.compileFunction dynamic-import hook is enabled. - execArgv: process.execArgv, - }); - } catch (e) { - try { - process.chdir(priorCwd); - } catch (_e) { - /* best-effort restore */ - } - resolve({ - id: job.id, - ok: false, - stdout: '', - stderr: '', - exit_code: null, - timed_out: false, - elapsed_ms: Date.now() - start, - error: 'failed to spawn worker thread: ' + (e && e.stack ? e.stack : String(e)), - }); - return; - } - - const outP = collect(worker.stdout); - const errP = collect(worker.stderr); - let exitCode = 0; - let timedOut = false; - let extraErr = ''; - - let timer = null; - if (job.timeout_ms && job.timeout_ms > 0) { - timer = setTimeout(() => { - timedOut = true; - worker.terminate(); - }, job.timeout_ms); - } - - worker.on('error', (e) => { - extraErr += (e && e.stack ? e.stack : String(e)) + '\n'; - if (!exitCode) exitCode = 1; - }); - - worker.on('exit', async (code) => { - if (timer) clearTimeout(timer); - // Restore the host cwd only now: a worker thread reads its cwd - // asynchronously as it initializes, so the host must stay at `job.cwd` - // for the worker's whole life. Jobs are serialized, so the next job - // starts from this restored (prior) directory rather than inheriting - // this one's. - try { - process.chdir(priorCwd); - } catch (_e) { - /* best-effort restore */ - } - if (code && !exitCode) exitCode = code; - const stdout = await outP; - const stderr = (await errP) + extraErr; - resolve({ - id: job.id, - ok: true, - stdout, - stderr, - exit_code: timedOut ? null : exitCode, - timed_out: timedOut, - elapsed_ms: Date.now() - start, - error: null, - }); - }); - }); -} - -let protocolInput = process.stdin; -let protocolStream = process.stdout; - -function reply(obj) { - protocolStream.write(JSON.stringify(obj) + '\n'); -} - -function serve() { - // Announce readiness, then serve jobs one at a time (the Rust pool already - // sends at most one outstanding job per worker; the chain keeps ordering). - reply({ - ready: true, - protocol: PROTOCOL_VERSION, - lang: 'node', - protocol_token: process.env.OPENHUMAN_RUNTIME_POOL_PROTOCOL_TOKEN || null, - }); - - const readline = require('readline'); - const rl = readline.createInterface({ input: protocolInput }); - let chain = Promise.resolve(); - rl.on('line', (line) => { - const trimmed = line.trim(); - if (!trimmed) return; - let job; - try { - job = JSON.parse(trimmed); - } catch (_e) { - return; // ignore unparseable lines - } - chain = chain - .then(() => runJob(job)) - .then((res) => reply(res)) - .catch((e) => { - reply({ - id: job && job.id, - ok: false, - stdout: '', - stderr: '', - exit_code: null, - timed_out: false, - elapsed_ms: 0, - error: String((e && e.stack) || e), - }); - }); - }); - rl.on('close', () => { - // Drain any in-flight job before exiting so a closed stdin doesn't drop work - // that was already accepted onto the chain. - Promise.resolve(chain).finally(() => process.exit(0)); - }); -} - -const protocolAddr = process.env.OPENHUMAN_RUNTIME_POOL_PROTOCOL_ADDR; -if (protocolAddr) { - const net = require('net'); - const split = protocolAddr.lastIndexOf(':'); - const host = protocolAddr.slice(0, split); - const port = Number(protocolAddr.slice(split + 1)); - const socket = net.createConnection({ host, port }); - socket.once('connect', () => { - protocolInput = socket; - protocolStream = socket; - serve(); - }); - socket.once('error', (e) => { - process.stderr.write('runtime pool protocol connection failed: ' + String(e) + '\n'); - process.exit(1); - }); -} else { - // Backward-compatible developer launch; production Node workers always use - // the isolated socket configured by Rust. - serve(); -} diff --git a/src/openhuman/runtime/pool/pool_worker.py b/src/openhuman/runtime/pool/pool_worker.py deleted file mode 100644 index 7346a95979..0000000000 --- a/src/openhuman/runtime/pool/pool_worker.py +++ /dev/null @@ -1,212 +0,0 @@ -# OpenHuman runtime-pool Python worker harness (issue #5106). -# -# A single long-lived `python` process that executes inline Python jobs for many -# skill runs, so the fleet pays one warm interpreter instead of one child per -# run. -# -# Protocol (newline-delimited JSON over an authenticated loopback socket, -# see runtime_pool/protocol.rs): -# 1. Print exactly one ready line: {"ready":true,"protocol":1,"lang":"python"} -# 2. For each request line {id,kind:"inline",code,cwd,timeout_ms} reply with -# {id,ok,stdout,stderr,exit_code,timed_out,elapsed_ms,error}. -# -# Each job runs in this interpreter with stdout/stderr redirected into buffers so -# a job's prints never corrupt the protocol stream. Isolation is per-job globals -# plus the pool's recycle-after-N-jobs; CPython cannot safely kill a running -# thread, so the soft deadline is best-effort SIGALRM on Unix and otherwise the -# Rust side's hard deadline kills + respawns the worker. - -import sys -import os -import json -import time -import tempfile -import traceback - -PROTOCOL_VERSION = 1 - -# Production workers use one authenticated duplex socket for protocol traffic, -# leaving fd 0 at EOF and fd 1/2 entirely available to job capture. The stdio -# fallback keeps the harness convenient to launch by hand. -_PROTOCOL_TOKEN = os.environ.get("OPENHUMAN_RUNTIME_POOL_PROTOCOL_TOKEN") -_PROTOCOL_ADDR = os.environ.get("OPENHUMAN_RUNTIME_POOL_PROTOCOL_ADDR") -_PROTOCOL_SOCKET = None -if _PROTOCOL_ADDR: - import socket - - _host, _port = _PROTOCOL_ADDR.rsplit(":", 1) - _PROTOCOL_SOCKET = socket.create_connection((_host, int(_port))) - _PROTO_IN = _PROTOCOL_SOCKET.makefile("r") - _PROTO = _PROTOCOL_SOCKET.makefile("w", buffering=1) -else: - # Private duplicates prevent per-job fd redirection from touching protocol. - _PROTO_IN = os.fdopen(os.dup(0), "r", buffering=1) - _PROTO = os.fdopen(os.dup(1), "w", buffering=1) - -try: - import signal - - _HAVE_ALARM = hasattr(signal, "SIGALRM") and hasattr(signal, "setitimer") -except Exception: # pragma: no cover - platform without signal - signal = None - _HAVE_ALARM = False - - -class _JobTimeout(Exception): - pass - - -def _run_job(job): - code = job.get("code") or "" - cwd = job.get("cwd") - timeout_ms = job.get("timeout_ms") - start = time.time() - exit_code = 0 - timed_out = False - extra_err = "" - - old_cwd = None - if cwd: - try: - old_cwd = os.getcwd() - os.chdir(cwd) - except Exception as exc: - # Match subprocess cwd semantics: if the requested action root - # cannot be entered, user code must not run in this long-lived - # worker's inherited directory. - return { - "id": job.get("id"), - "ok": False, - "stdout": "", - "stderr": "", - "exit_code": None, - "timed_out": False, - "elapsed_ms": int((time.time() - start) * 1000), - "error": f"failed to set worker cwd: {exc!r}", - } - - # Capture at the FILE-DESCRIPTOR level (not just `sys.stdout`) so - # `os.write(1, ...)`, subprocesses, and native extensions are captured too — - # otherwise they would leak onto the real stdout, which is the NDJSON - # protocol channel. Temp files (vs pipes) avoid buffer-deadlock on large - # output. - in_f = tempfile.TemporaryFile(mode="w+b") - out_f = tempfile.TemporaryFile(mode="w+b") - err_f = tempfile.TemporaryFile(mode="w+b") - saved_in = os.dup(0) - saved_out = os.dup(1) - saved_err = os.dup(2) - os.dup2(in_f.fileno(), 0) - os.dup2(out_f.fileno(), 1) - os.dup2(err_f.fileno(), 2) - - armed = False - if _HAVE_ALARM and timeout_ms and timeout_ms > 0: - def _on_alarm(_signum, _frame): - raise _JobTimeout() - - signal.signal(signal.SIGALRM, _on_alarm) - signal.setitimer(signal.ITIMER_REAL, timeout_ms / 1000.0) - armed = True - - try: - # Fresh globals per job so top-level names don't leak between runs. - g = {"__name__": "__main__", "__builtins__": __builtins__} - exec(compile(code, "", "exec"), g, g) - except _JobTimeout: - timed_out = True - except SystemExit as e: # honour sys.exit(n) - if e.code is None: - exit_code = 0 - elif isinstance(e.code, int): - exit_code = e.code - else: - exit_code = 1 - extra_err = str(e.code) + "\n" - except BaseException: # noqa: BLE001 - surface any job failure to the caller - exit_code = 1 - extra_err = traceback.format_exc() - finally: - if armed: - signal.setitimer(signal.ITIMER_REAL, 0) - # Flush Python's buffers to the redirected fds, then restore the real - # stdout/stderr before reading the captures. A flush failure is surfaced - # in the job's stderr rather than silently discarded. - try: - sys.stdout.flush() - except Exception as flush_err: # noqa: BLE001 - extra_err += f"[harness] stdout flush failed: {flush_err!r}\n" - try: - sys.stderr.flush() - except Exception as flush_err: # noqa: BLE001 - extra_err += f"[harness] stderr flush failed: {flush_err!r}\n" - os.dup2(saved_in, 0) - os.dup2(saved_out, 1) - os.dup2(saved_err, 2) - os.close(saved_in) - os.close(saved_out) - os.close(saved_err) - if old_cwd is not None: - try: - os.chdir(old_cwd) - except Exception: - pass - - in_f.close() - out_f.seek(0) - err_f.seek(0) - stdout = out_f.read().decode("utf-8", "replace") - stderr = err_f.read().decode("utf-8", "replace") + extra_err - out_f.close() - err_f.close() - - return { - "id": job.get("id"), - "ok": True, - "stdout": stdout, - "stderr": stderr, - "exit_code": None if timed_out else exit_code, - "timed_out": timed_out, - "elapsed_ms": int((time.time() - start) * 1000), - "error": None, - } - - -def _reply(obj): - _PROTO.write(json.dumps(obj) + "\n") - _PROTO.flush() - - -def main(): - _reply({ - "ready": True, - "protocol": PROTOCOL_VERSION, - "lang": "python", - "protocol_token": _PROTOCOL_TOKEN, - }) - for line in _PROTO_IN: - line = line.strip() - if not line: - continue - try: - job = json.loads(line) - except Exception: - continue # ignore unparseable lines - try: - res = _run_job(job) - except Exception as e: # harness-level failure - res = { - "id": job.get("id") if isinstance(job, dict) else None, - "ok": False, - "stdout": "", - "stderr": "", - "exit_code": None, - "timed_out": False, - "elapsed_ms": 0, - "error": repr(e), - } - _reply(res) - - -if __name__ == "__main__": - main() diff --git a/src/openhuman/runtime/pool/protocol.rs b/src/openhuman/runtime/pool/protocol.rs deleted file mode 100644 index 1b10755709..0000000000 --- a/src/openhuman/runtime/pool/protocol.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! Wire protocol between the core and a pooled language worker. -//! -//! Newline-delimited JSON over a per-worker duplex transport, mirroring the -//! [`runtime_python_server`](crate::openhuman::runtime::python_server) protocol. -//! Production workers use an authenticated loopback socket so job fd 0/1/2 -//! cannot consume or corrupt protocol traffic: -//! -//! 1. On startup the worker prints exactly one [`PoolReadyLine`]. -//! 2. The core writes one [`PoolJobRequest`] per line; the worker replies with -//! one [`PoolJobResponse`] per line, correlated by `id`. -//! -//! Child stdout/stderr are drained separately and never carry protocol frames. - -use serde::{Deserialize, Serialize}; - -/// Bumped whenever the request/response shape changes incompatibly. The worker -/// echoes the version it speaks in its ready line; a mismatch fails the launch. -pub const PROTOCOL_VERSION: u32 = 1; - -/// Handshake line printed once by the worker on startup. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PoolReadyLine { - #[serde(default)] - pub ready: bool, - #[serde(default)] - pub protocol: Option, - /// `"node"` / `"python"` — a sanity check that the right harness launched. - #[serde(default)] - pub lang: Option, - #[serde(default)] - pub error: Option, - /// Optional per-launch secret used when the protocol travels over an - /// isolated loopback socket instead of stdout. - #[serde(default)] - pub protocol_token: Option, -} - -/// A single unit of work sent to a worker. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PoolJobRequest { - /// Correlation id — the worker echoes it back in the response. - pub id: String, - /// Job kind. Today only `"inline"` (evaluate `code`); reserved for future - /// `"script"` support. - pub kind: String, - /// Inline source to evaluate (for `kind == "inline"`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub code: Option, - /// Working directory for the job. The worker `chdir`s per job so relative - /// paths resolve against the caller's action sandbox. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub cwd: Option, - /// Soft per-job deadline in milliseconds. The worker aborts the job when it - /// elapses and replies with `timed_out = true`. Absent ⇒ run to completion. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub timeout_ms: Option, -} - -/// A worker's reply to a [`PoolJobRequest`]. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PoolJobResponse { - pub id: Option, - /// `true` when the harness ran the job to a normal conclusion (the user - /// code may still have thrown — see `exit_code`/`stderr`). `false` only for - /// harness-level failures described in `error`. - #[serde(default)] - pub ok: bool, - #[serde(default)] - pub stdout: String, - #[serde(default)] - pub stderr: String, - /// `0` on clean completion, non-zero when the job threw/exited non-zero, - /// `None` when not applicable. - #[serde(default)] - pub exit_code: Option, - /// Set when the worker aborted the job at its soft deadline. - #[serde(default)] - pub timed_out: bool, - #[serde(default)] - pub elapsed_ms: u64, - /// Harness-level error (worker could not run the job at all). - #[serde(default)] - pub error: Option, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn ready_line_parses() { - let ready: PoolReadyLine = - serde_json::from_str(r#"{"ready":true,"protocol":1,"lang":"node"}"#).unwrap(); - assert!(ready.ready); - assert_eq!(ready.protocol, Some(PROTOCOL_VERSION)); - assert_eq!(ready.lang.as_deref(), Some("node")); - } - - #[test] - fn request_omits_absent_optional_fields() { - let req = PoolJobRequest { - id: "3".to_string(), - kind: "inline".to_string(), - code: Some("console.log(1)".to_string()), - cwd: None, - timeout_ms: None, - }; - let line = serde_json::to_string(&req).unwrap(); - assert!(line.contains("\"kind\":\"inline\"")); - assert!(!line.contains("cwd")); - assert!(!line.contains("timeout_ms")); - } - - #[test] - fn response_parses_failure_envelope() { - let resp: PoolJobResponse = serde_json::from_str( - r#"{"id":"7","ok":true,"stdout":"","stderr":"boom","exit_code":1,"elapsed_ms":12}"#, - ) - .unwrap(); - assert!(resp.ok); - assert_eq!(resp.exit_code, Some(1)); - assert_eq!(resp.stderr, "boom"); - assert!(!resp.timed_out); - } -} diff --git a/src/openhuman/runtime/pool/types.rs b/src/openhuman/runtime/pool/types.rs index c8392c911e..8adb93a874 100644 --- a/src/openhuman/runtime/pool/types.rs +++ b/src/openhuman/runtime/pool/types.rs @@ -1,64 +1,116 @@ -//! Public types callers of the runtime pool see. +//! The types callers of pooled execution see. use std::time::Duration; +use tinyruntime_bus::{ExecResponse, Language}; + use crate::openhuman::config::RuntimePoolLangConfig; -/// Language a pool serves. Drives which harness script + interpreter launches. +/// A language with a worker pool. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PoolLang { + /// JavaScript on the managed Node.js toolchain. Node, + /// Python. Python, } impl PoolLang { + /// The identifier used in logs and status surfaces. + #[must_use] pub fn id(self) -> &'static str { match self { - PoolLang::Node => "node", - PoolLang::Python => "python", + Self::Node => "node", + Self::Python => "python", + } + } + + /// The bus language this pool serves. + #[must_use] + pub fn language(self) -> Language { + match self { + Self::Node => Language::nodejs(), + Self::Python => Language::python(), + } + } + + /// The pool a bus language belongs to, if this build has one for it. + /// + /// A language this core ships no pool concept for is `None` rather than an + /// error: the module routes whatever its configuration routes, and a status + /// surface should skip an unfamiliar entry rather than fail rendering. + #[must_use] + pub fn from_language(language: &Language) -> Option { + match language.as_str() { + tinyruntime_bus::NODEJS => Some(Self::Node), + tinyruntime_bus::PYTHON => Some(Self::Python), + _ => None, } } } -/// The result of running one job on a pooled worker. Mirrors the fields a -/// per-call `std::process` spawn would have exposed, plus `queue_wait` so -/// callers can surface backpressure in run logs (a DoD requirement of #5106). +/// The result of running one job on a pooled worker. +/// +/// Mirrors the fields a per-call spawn would have exposed, plus `queue_wait` so +/// callers can surface backpressure in run logs — a host that cannot tell a slow +/// job from a busy pool will tune the wrong thing. #[derive(Debug, Clone)] pub struct PoolExecOutcome { + /// Everything the job wrote to standard output. pub stdout: String, + /// Everything the job wrote to standard error. pub stderr: String, - /// `0` on success, non-zero when the job threw / exited non-zero. + /// `0` on success, non-zero when the job threw or exited non-zero. pub exit_code: Option, /// The job hit its soft deadline and was aborted. pub timed_out: bool, /// Wall-clock the job itself took inside the worker. pub elapsed: Duration, - /// How long the submission waited for a free worker (queue backpressure). + /// How long the submission waited for a free worker. pub queue_wait: Duration, } impl PoolExecOutcome { + /// Adapt a module reply. + #[must_use] + pub fn from_module(response: &ExecResponse) -> Self { + Self { + stdout: response.stdout.clone(), + stderr: response.stderr.clone(), + exit_code: response.exit_code, + timed_out: response.timed_out, + elapsed: Duration::from_millis(response.elapsed_ms), + queue_wait: Duration::from_millis(response.queue_wait_ms), + } + } + /// A job "succeeded" when it ran to completion with a zero (or absent) exit /// code and did not time out. + #[must_use] pub fn success(&self) -> bool { !self.timed_out && matches!(self.exit_code, None | Some(0)) } } -/// The knobs a single language pool reads from config, snapshotted at pool -/// construction. Kept as a plain owned struct so the pool never re-reads config -/// mid-flight. +/// The knobs a single language pool reads from config. +/// +/// Kept as a plain owned struct so a caller never re-reads config mid-flight. #[derive(Debug, Clone)] pub struct PoolSettings { + /// Concurrent workers. pub max_workers: usize, + /// Retire a worker after this long idle, or never. pub idle_ttl: Option, + /// Retire a worker after this many jobs. `0` disables recycling. pub recycle_after_jobs: u64, + /// Jobs allowed to queue beyond the worker slots. pub max_queue_depth: usize, } impl PoolSettings { /// Derive the effective settings from a per-language config block, applying /// the same "never zero" clamps the config getters use. + #[must_use] pub fn from_lang_config(cfg: &RuntimePoolLangConfig) -> Self { Self { max_workers: cfg.effective_max_workers(), @@ -74,44 +126,5 @@ impl PoolSettings { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn outcome_success_semantics() { - let base = PoolExecOutcome { - stdout: String::new(), - stderr: String::new(), - exit_code: Some(0), - timed_out: false, - elapsed: Duration::ZERO, - queue_wait: Duration::ZERO, - }; - assert!(base.success()); - assert!(!PoolExecOutcome { - exit_code: Some(1), - ..base.clone() - } - .success()); - assert!(!PoolExecOutcome { - timed_out: true, - ..base.clone() - } - .success()); - } - - #[test] - fn settings_disable_idle_reap_on_zero() { - let cfg = RuntimePoolLangConfig { - enabled: Some(true), - max_workers: 3, - idle_ttl_secs: 0, - recycle_after_jobs: 5, - max_queue_depth: 10, - }; - let s = PoolSettings::from_lang_config(&cfg); - assert_eq!(s.max_workers, 3); - assert!(s.idle_ttl.is_none()); - assert_eq!(s.recycle_after_jobs, 5); - } -} +#[path = "types_tests.rs"] +mod tests; diff --git a/src/openhuman/runtime/pool/worker.rs b/src/openhuman/runtime/pool/worker.rs deleted file mode 100644 index 2370be3304..0000000000 --- a/src/openhuman/runtime/pool/worker.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! A single pooled worker: one long-lived interpreter child speaking the -//! newline-delimited JSON [`protocol`](super::protocol) over an isolated -//! loopback socket (with a stdio fallback for development harnesses). -//! -//! One worker runs **one job at a time**; concurrency comes from the -//! [`LangPool`](super::pool::LangPool) holding several workers. A worker stays -//! warm between jobs (the whole point — no per-run interpreter spawn) until it -//! is idle-reaped or recycled after N jobs. - -use std::path::PathBuf; -use std::process::Stdio; -use std::time::{Duration, Instant}; - -use anyhow::{bail, Context, Result}; -use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader, Lines}; -use tokio::net::TcpListener; -use tokio::process::{Child, ChildStderr, ChildStdout, Command}; - -use super::protocol::{PoolJobRequest, PoolJobResponse, PoolReadyLine, PROTOCOL_VERSION}; -use super::types::PoolLang; - -/// How long to wait for a freshly-spawned worker to print its ready line. -const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); - -/// Everything needed to (re)spawn a worker for one language. Cheap to clone so -/// the pool can respawn on demand. -#[derive(Debug, Clone)] -pub struct WorkerLaunch { - pub lang: PoolLang, - /// Interpreter binary (`node` / `python`). - pub bin: PathBuf, - /// Args after the binary — typically `[harness_script_path]`. - pub args: Vec, - /// Full environment for the child (already allow-listed by the backend). - /// The child's env is cleared first, so this is the complete set. - pub env: Vec<(String, String)>, - /// Keep user fd 0/1/2 away from the NDJSON request/response stream by - /// serving the protocol over a per-launch authenticated loopback socket. - pub isolated_protocol: bool, -} - -/// Failure from [`PoolWorker::submit`], tagged with whether the job was already -/// dispatched to the worker. A retry / legacy fallback is only safe when the job -/// was **not** dispatched (it never ran); a post-dispatch failure is terminal so -/// the same job is never executed twice. -#[derive(Debug)] -pub struct SubmitError { - pub err: anyhow::Error, - pub dispatched: bool, -} - -impl SubmitError { - fn pre(err: anyhow::Error) -> Self { - Self { - err, - dispatched: false, - } - } - fn post(err: anyhow::Error) -> Self { - Self { - err, - dispatched: true, - } - } -} - -impl std::fmt::Display for SubmitError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:#}", self.err) - } -} - -/// A warm interpreter child plus its bookkeeping. -pub struct PoolWorker { - launch: WorkerLaunch, - _child: Child, - stdin: Box, - responses: Lines>>, - jobs_done: u64, - last_used: Instant, -} - -impl PoolWorker { - pub fn jobs_done(&self) -> u64 { - self.jobs_done - } - - pub fn last_used(&self) -> Instant { - self.last_used - } - - /// Spawn a new worker and complete the readiness handshake. - pub async fn spawn(launch: &WorkerLaunch) -> Result { - tracing::info!( - lang = launch.lang.id(), - bin = %launch.bin.display(), - "[runtime_pool] spawning worker" - ); - let mut cmd = Command::new(&launch.bin); - cmd.args(&launch.args); - cmd.env_clear(); - for (key, value) in &launch.env { - cmd.env(key, value); - } - let isolated_protocol = if launch.isolated_protocol { - let listener = TcpListener::bind(("127.0.0.1", 0)) - .await - .context("binding isolated worker protocol listener")?; - let addr = listener - .local_addr() - .context("reading isolated worker protocol address")?; - let token = uuid::Uuid::new_v4().to_string(); - cmd.env("OPENHUMAN_RUNTIME_POOL_PROTOCOL_ADDR", addr.to_string()); - cmd.env("OPENHUMAN_RUNTIME_POOL_PROTOCOL_TOKEN", &token); - Some((listener, token)) - } else { - None - }; - if isolated_protocol.is_some() { - // Jobs inherit EOF on fd 0, matching Command::output(), while the - // harness receives requests over the isolated duplex socket. - cmd.stdin(Stdio::null()); - } else { - cmd.stdin(Stdio::piped()); - } - cmd.stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true); - // Suppress the Windows console flash for each spawned worker. - #[cfg(windows)] - { - use std::os::windows::process::CommandExt; - const CREATE_NO_WINDOW: u32 = 0x0800_0000; - cmd.creation_flags(CREATE_NO_WINDOW); - } - - let mut child = cmd - .spawn() - .with_context(|| format!("spawning {} worker", launch.lang.id()))?; - let child_stdin = child.stdin.take(); - let stdout = child.stdout.take().context("worker stdout missing")?; - if let Some(stderr) = child.stderr.take() { - drain_stderr(launch.lang, stderr); - } - let (stdin, reader, expected_token): ( - Box, - Box, - Option, - ) = if let Some((listener, token)) = isolated_protocol { - // stdout is now exclusively user fd-level output. Drain it so - // chatty jobs cannot block; protocol frames use the socket. - drain_stdout(launch.lang, stdout); - let (stream, _) = tokio::time::timeout(HANDSHAKE_TIMEOUT, listener.accept()) - .await - .map_err(|_| { - anyhow::anyhow!("{} worker protocol connection timed out", launch.lang.id()) - })? - .context("accepting isolated worker protocol connection")?; - let (reader, writer) = tokio::io::split(stream); - (Box::new(writer), Box::new(reader), Some(token)) - } else { - ( - Box::new(child_stdin.context("worker stdin missing")?), - Box::new(stdout), - None, - ) - }; - let mut lines = BufReader::new(reader).lines(); - - let ready_line = match tokio::time::timeout(HANDSHAKE_TIMEOUT, lines.next_line()).await { - Ok(Ok(Some(line))) => line, - Ok(Ok(None)) => bail!( - "{} worker exited before readiness handshake", - launch.lang.id() - ), - Ok(Err(error)) => { - return Err(error).context("reading worker handshake"); - } - Err(_) => bail!("{} worker readiness handshake timed out", launch.lang.id()), - }; - let ready: PoolReadyLine = serde_json::from_str(&ready_line) - .with_context(|| format!("parsing worker ready line: {ready_line}"))?; - if !ready.ready { - bail!( - "{} worker failed to start: {}", - launch.lang.id(), - ready.error.unwrap_or_else(|| "unknown".to_string()) - ); - } - if ready.protocol != Some(PROTOCOL_VERSION) { - bail!( - "{} worker protocol mismatch: expected {}, got {:?}", - launch.lang.id(), - PROTOCOL_VERSION, - ready.protocol - ); - } - if ready.protocol_token != expected_token { - bail!("{} worker protocol authentication failed", launch.lang.id()); - } - tracing::info!(lang = launch.lang.id(), "[runtime_pool] worker ready"); - - Ok(Self { - launch: launch.clone(), - _child: child, - stdin, - responses: lines, - jobs_done: 0, - last_used: Instant::now(), - }) - } - - /// Submit one job and await its response. - /// - /// `hard_timeout` is a **safety net** above the worker's own soft deadline: - /// the worker aborts a job at `req.timeout_ms` and still replies, so this - /// only fires if the worker itself has wedged. On `Err` the caller must - /// discard this worker — its stdio framing can no longer be trusted. - pub async fn submit( - &mut self, - req: &PoolJobRequest, - hard_timeout: Option, - ) -> std::result::Result { - let mut line = serde_json::to_string(req) - .map_err(|e| SubmitError::pre(anyhow::Error::new(e).context("serialising pool job")))?; - line.push('\n'); - // A write failure means the bytes never reached the worker (e.g. a - // reused idle worker died) → the job did not run → safe to retry. - self.stdin.write_all(line.as_bytes()).await.map_err(|e| { - SubmitError::pre(anyhow::Error::new(e).context("writing pool job request")) - })?; - // Past this point the request bytes are in the pipe: the job may execute, - // so any later failure is terminal (never re-run the same job). - self.stdin.flush().await.map_err(|e| { - SubmitError::post(anyhow::Error::new(e).context("flushing pool job request")) - })?; - - // Fixed deadline: `continue`ing over unparseable / mismatched-id lines - // must NOT reset the wedged-worker timeout, so it bounds the total wait. - let deadline = hard_timeout.map(|t| tokio::time::Instant::now() + t); - loop { - let next = match deadline { - Some(dl) => match tokio::time::timeout_at(dl, self.responses.next_line()).await { - Ok(inner) => inner, - Err(_) => { - return Err(SubmitError::post(anyhow::anyhow!( - "pool worker job timed out (hard deadline; worker wedged)" - ))) - } - }, - None => self.responses.next_line().await, - }; - let line = match next { - Ok(Some(line)) => line, - Ok(None) => { - return Err(SubmitError::post(anyhow::anyhow!( - "pool worker closed stdout" - ))) - } - Err(error) => { - return Err(SubmitError::post( - anyhow::Error::new(error).context("reading pool job response"), - )) - } - }; - let response: PoolJobResponse = match serde_json::from_str(&line) { - Ok(response) => response, - Err(error) => { - tracing::warn!( - lang = self.launch.lang.id(), - "[runtime_pool] unparseable worker line skipped: {error}" - ); - continue; - } - }; - if response.id.as_deref() != Some(req.id.as_str()) { - tracing::debug!( - lang = self.launch.lang.id(), - "[runtime_pool] skipped response for different id={:?}", - response.id - ); - continue; - } - self.jobs_done += 1; - self.last_used = Instant::now(); - return Ok(response); - } - } - - /// Whether this worker has served enough jobs to be recycled. `0` disables. - pub fn should_recycle(&self, recycle_after: u64) -> bool { - recycle_due(self.jobs_done, recycle_after) - } - - /// Whether this worker has been idle at least `ttl`. - pub fn idle_expired(&self, ttl: Duration) -> bool { - idle_due(self.last_used.elapsed(), ttl) - } - - /// Signal the child to exit. Best-effort; `kill_on_drop` is the backstop. - pub fn shutdown(mut self) { - if let Err(error) = self._child.start_kill() { - tracing::debug!( - lang = self.launch.lang.id(), - "[runtime_pool] failed to signal worker shutdown: {error}" - ); - } - } -} - -/// Continuously drain a worker's stderr so a chatty child never blocks on a -/// full pipe. Lines are logged at trace; never parsed as protocol. -fn drain_stderr(lang: PoolLang, stderr: ChildStderr) { - tokio::spawn(async move { - let mut reader = BufReader::new(stderr).lines(); - while let Ok(Some(line)) = reader.next_line().await { - tracing::trace!(lang = lang.id(), "[runtime_pool] worker stderr: {line}"); - } - }); -} - -/// Drain fd-level stdout from workers whose protocol uses an isolated socket. -/// This output is deliberately never parsed as NDJSON, so user code cannot -/// forge a response frame or desynchronise subsequent jobs. -fn drain_stdout(lang: PoolLang, stdout: ChildStdout) { - tokio::spawn(async move { - let mut reader = BufReader::new(stdout).lines(); - while let Ok(Some(line)) = reader.next_line().await { - tracing::trace!(lang = lang.id(), "[runtime_pool] worker fd stdout: {line}"); - } - }); -} - -/// Pure recycle predicate: a worker is due for recycling once it has served -/// `recycle_after` jobs (`0` disables recycling). -fn recycle_due(jobs_done: u64, recycle_after: u64) -> bool { - recycle_after > 0 && jobs_done >= recycle_after -} - -/// Pure idle-expiry predicate: idle for at least `ttl`. -fn idle_due(idle_elapsed: Duration, ttl: Duration) -> bool { - idle_elapsed >= ttl -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn recycle_due_respects_budget_and_disable() { - assert!(!recycle_due(0, 0), "recycle_after=0 disables recycling"); - assert!(!recycle_due(100, 0), "recycle_after=0 never recycles"); - assert!(!recycle_due(4, 5), "below budget"); - assert!(recycle_due(5, 5), "at budget"); - assert!(recycle_due(6, 5), "past budget"); - } - - #[test] - fn idle_due_is_inclusive_at_ttl() { - assert!(!idle_due(Duration::from_secs(4), Duration::from_secs(5))); - assert!(idle_due(Duration::from_secs(5), Duration::from_secs(5))); - assert!(idle_due(Duration::from_secs(6), Duration::from_secs(5))); - } - - #[test] - fn submit_error_tags_dispatch_state() { - let pre = SubmitError::pre(anyhow::anyhow!("write failed")); - assert!( - !pre.dispatched, - "write failures are pre-dispatch (retryable)" - ); - let post = SubmitError::post(anyhow::anyhow!("read timed out")); - assert!( - post.dispatched, - "read failures are post-dispatch (terminal)" - ); - } -} From 483709418aa73c7ef86b66f338ed2f9a9eb4c304 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:10:51 +0300 Subject: [PATCH 16/55] refactor(pool): delegate node and python pool backends to a shared dispatch Replace the per-language pool backends with thin wrappers that call a common `super::run_inline` dispatch, removing the duplicated worker-launch logic, harness materialisation, and inline test suites from `node.rs` and `python.rs`. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/pool/node.rs | 365 +--------------------- src/openhuman/runtime/pool/pool_tests.rs | 83 +++++ src/openhuman/runtime/pool/python.rs | 246 ++------------- src/openhuman/runtime/pool/types_tests.rs | 83 +++++ 4 files changed, 205 insertions(+), 572 deletions(-) create mode 100644 src/openhuman/runtime/pool/pool_tests.rs create mode 100644 src/openhuman/runtime/pool/types_tests.rs diff --git a/src/openhuman/runtime/pool/node.rs b/src/openhuman/runtime/pool/node.rs index 40761c9482..347a84dcc3 100644 --- a/src/openhuman/runtime/pool/node.rs +++ b/src/openhuman/runtime/pool/node.rs @@ -1,364 +1,33 @@ -//! Node.js pool backend: resolve the interpreter, materialise the JS harness, -//! and submit inline jobs to the shared [`LangPool`](super::pool::LangPool). +//! Node.js pooled execution: whether this host wants it, and how to ask for it. -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::Duration; -use anyhow::Result; -use tokio::sync::OnceCell; +use tinyruntime_bus::Language; -use super::pool; -use super::types::{PoolExecOutcome, PoolLang, PoolSettings}; -use super::worker::WorkerLaunch; -use crate::openhuman::config::{RuntimePoolConfig, RuntimePoolLangConfig}; +use super::{PoolExecOutcome, PoolRunError}; +use crate::openhuman::config::{Config, RuntimePoolConfig}; -/// The bundled Node worker harness (runs each inline job in an isolated -/// `worker_thread`, capturing its stdout/stderr and honouring a soft deadline). -const WORKER_JS: &str = include_str!("pool_worker.js"); - -/// Written once per process (see [`super::ensure_worker_script`]). -static NODE_SCRIPT: OnceCell = OnceCell::const_new(); - -/// Whether inline `node` jobs should route through the pool. `node.enabled` is -/// already implied by the tool only being constructed when the node runtime is -/// enabled, so only the pool switches are checked here. +/// Whether inline `node` jobs should route through the pool. +/// +/// Node defaults **on**: each job runs in its own `worker_thread`, so reuse is +/// safe — a fresh module graph and fresh globals per job. +#[must_use] pub fn enabled(pool: &RuntimePoolConfig) -> bool { - // Node defaults ON: each job runs in an isolated worker_thread, so reuse is - // safe (fresh module graph + globals per job). pool.enabled && pool.node.is_enabled(true) } /// Run inline JavaScript on a pooled, warm `node` worker. /// -/// `node_bin` / `bin_dir` come from the caller's already-resolved -/// [`ResolvedNode`](crate::openhuman::runtime::node::ResolvedNode). `workspace_dir` -/// and `lang_cfg` are injected at tool construction so this hot path never -/// re-reads config or re-writes the harness. `cwd` is the job's working -/// directory; `timeout` is the soft per-job deadline (`None` ⇒ run to completion). +/// # Errors +/// +/// [`PoolRunError`], classified so the caller knows whether falling back to a +/// per-call spawn is safe. pub async fn run_inline( - workspace_dir: &Path, - lang_cfg: &RuntimePoolLangConfig, - node_bin: &Path, - bin_dir: &Path, + config: &Config, code: String, cwd: Option, timeout: Option, -) -> Result { - let script = - super::ensure_worker_script(&NODE_SCRIPT, workspace_dir, "pool_worker.js", WORKER_JS) - .await - .map_err(super::pool::PoolRunError::PreDispatch)? - .to_string_lossy() - .into_owned(); - - let env = super::base_env(bin_dir); - - let launch = WorkerLaunch { - lang: PoolLang::Node, - bin: node_bin.to_path_buf(), - // `--experimental-vm-modules` lets the harness root dynamic `import()` at - // the job cwd (parity with `node -e`); the flag propagates to the per-job - // worker_thread via inherited `execArgv`. - args: vec![ - "--experimental-vm-modules".to_string(), - "--experimental-import-meta-resolve".to_string(), - script, - ], - env, - isolated_protocol: true, - }; - let settings = PoolSettings::from_lang_config(lang_cfg); - let pool = pool::ensure_pool(launch, settings).await; - - let cwd = cwd.map(|p| p.to_string_lossy().into_owned()); - pool.run_inline(code, cwd, timeout).await -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::openhuman::config::Config; - use crate::openhuman::runtime::pool::{all_stats, PoolLang}; - - /// Resolve the host `node` binary + its bin dir, or `None` to skip. - fn system_node() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let out = std::process::Command::new("node") - .args(["-e", "process.stdout.write(process.execPath)"]) - .output() - .ok()?; - if !out.status.success() { - return None; - } - let node_bin = std::path::PathBuf::from(String::from_utf8_lossy(&out.stdout).trim()); - let bin_dir = node_bin.parent()?.to_path_buf(); - Some((node_bin, bin_dir)) - } - - async fn node_spawns() -> u64 { - all_stats() - .await - .into_iter() - .find(|(lang, _)| *lang == PoolLang::Node) - .map(|(_, stats)| stats.worker_spawns) - .unwrap_or(0) - } - - /// End-to-end: two inline jobs run on the pool, share ONE warm worker, and - /// surface stdout / exit codes exactly like the legacy path. Skips when no - /// system `node` is available (keeps CI hermetic on node-less runners). - #[tokio::test] - async fn pooled_node_runs_inline_and_reuses_worker() { - let Some((node_bin, bin_dir)) = system_node() else { - eprintln!("[runtime_pool] test skipped: no system node on PATH"); - return; - }; - let tmp = std::env::temp_dir().join(format!("rt-pool-node-e2e-{}", std::process::id())); - std::fs::create_dir_all(&tmp).unwrap(); - - let mut config = Config::default(); - config.workspace_dir = tmp.clone(); - config.runtime_pool.node.max_workers = 1; - config.runtime_pool.node.recycle_after_jobs = 0; // no recycle mid-test - let lang = config.runtime_pool.node.clone(); - - let spawns_before = node_spawns().await; - - let out1 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "console.log(JSON.stringify({ v: 6 * 7 }))".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("job 1 runs"); - assert!(out1.success(), "job 1 should succeed: {out1:?}"); - assert!( - out1.stdout.contains("\"v\":42"), - "stdout was {:?}", - out1.stdout - ); - - let out2 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "throw new Error('nope')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("job 2 runs"); - assert!(!out2.success(), "throwing job should fail"); - assert!(out2.stderr.contains("nope"), "stderr was {:?}", out2.stderr); - - // cwd correctness: relative fs must resolve against the job's cwd, not - // the worker process's launch dir. Guards the host-chdir fix (a worker - // thread cannot chdir itself). Regression here = broken action sandbox. - std::fs::write(tmp.join("probe.txt"), "REL_OK").unwrap(); - let out3 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "const fs=require('fs'); process.stdout.write(fs.readFileSync('./probe.txt','utf8'))" - .to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("job 3 runs"); - assert!(out3.success(), "cwd-relative read should succeed: {out3:?}"); - assert_eq!(out3.stdout, "REL_OK", "relative read resolved wrong cwd"); - - // fd-level writes must never share the protocol transport. This forged - // frame uses the real request id; on stdout-based framing Rust would - // accept it instead of the harness response and desynchronise the next - // job. - let out4 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - r#" -const fs = require('fs'); -const { workerData } = require('worker_threads'); -fs.writeSync(1, JSON.stringify({ - id: workerData.id, - ok: true, - stdout: 'FORGED', - stderr: '', - exit_code: 0, - elapsed_ms: 0 -}) + '\n'); -console.log('REAL_RESPONSE'); -"# - .to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("fd-level output job runs"); - assert!( - out4.success(), - "fd-level output job should succeed: {out4:?}" - ); - assert_eq!(out4.stdout, "REAL_RESPONSE\n"); - - // Bare dynamic imports retain node -e semantics rather than being - // rewritten to a cwd-relative file URL. - let out5 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "const fs = await import('fs'); console.log(typeof fs.readFileSync)".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("bare import job runs"); - assert!(out5.success(), "bare import should succeed: {out5:?}"); - assert_eq!(out5.stdout, "function\n"); - - // ESM-only packages expose an `import` condition but no CommonJS - // `require` condition. Bare imports must therefore use Node's ESM - // resolver rooted at the job cwd. - let esm_package = tmp.join("node_modules/esm-only"); - std::fs::create_dir_all(&esm_package).unwrap(); - std::fs::write( - esm_package.join("package.json"), - r#"{"name":"esm-only","type":"module","exports":{"import":"./index.mjs"}}"#, - ) - .unwrap(); - std::fs::write( - esm_package.join("index.mjs"), - "export const marker = 'ESM_ONLY_OK';", - ) - .unwrap(); - let esm_out = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "const { marker } = await import('esm-only'); console.log(marker)".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("ESM-only package import runs"); - assert!( - esm_out.success(), - "ESM-only package import should succeed: {esm_out:?}" - ); - assert_eq!(esm_out.stdout, "ESM_ONLY_OK\n"); - - // vm dynamic-import hooks receive attributes separately; forward them - // so JSON modules preserve legacy node -e behavior. - std::fs::write(tmp.join("data.json"), r#"{"answer":42}"#).unwrap(); - let json_out = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "const data = await import('./data.json', { with: { type: 'json' } }); console.log(data.default.answer)" - .to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("JSON import with attributes runs"); - assert!( - json_out.success(), - "JSON import with attributes should succeed: {json_out:?}" - ); - assert_eq!(json_out.stdout, "42\n"); - - // User warnings are tool output, not harness noise. Never suppress them - // globally on the pooled worker. - let warning_out = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "process.emitWarning('POOL_WARNING')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("warning job runs"); - assert!(warning_out.success(), "warning job failed: {warning_out:?}"); - assert!( - warning_out.stderr.contains("POOL_WARNING"), - "user warning was hidden: {:?}", - warning_out.stderr - ); - - // The protocol never shares fd 0 with user code. Legacy Command::output - // supplies EOF on stdin, so pooled code must do the same rather than - // blocking on or consuming the next NDJSON request. - let stdin_out = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "const fs=require('fs'); console.log(JSON.stringify(fs.readFileSync(0,'utf8')))" - .to_string(), - Some(tmp.clone()), - Some(Duration::from_secs(2)), - ) - .await - .expect("stdin EOF job runs"); - assert!(stdin_out.success(), "stdin should be EOF: {stdin_out:?}"); - assert_eq!(stdin_out.stdout, "\"\"\n"); - - // A missing cwd is a harness error. Running in the worker's inherited - // cwd would escape the requested action root and diverge from legacy - // Command::current_dir behavior. - let missing_cwd = tmp.join("deleted-action-root"); - let err = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "require('fs').writeFileSync('must-not-exist.txt', 'bad')".to_string(), - Some(missing_cwd.clone()), - None, - ) - .await - .expect_err("missing cwd must fail closed"); - assert!( - err.to_string().contains("failed to set worker cwd"), - "unexpected missing-cwd error: {err}" - ); - assert!(!tmp.join("must-not-exist.txt").exists()); - - // The harness-level cwd error must not poison framing for the next job. - let out6 = run_inline( - &config.workspace_dir, - &lang, - &node_bin, - &bin_dir, - "console.log('AFTER_CWD_ERROR')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("job after cwd error runs"); - assert_eq!(out6.stdout, "AFTER_CWD_ERROR\n"); - - // At most one NEW worker spawned for all jobs ⇒ the warm worker was - // reused. Measured as a delta so prior global pool state can't skew it. - let spawns_after = node_spawns().await; - assert!( - spawns_after - spawns_before <= 1, - "expected warm-worker reuse: {} new spawns", - spawns_after - spawns_before - ); - - let _ = std::fs::remove_dir_all(&tmp); - } +) -> Result { + super::run_inline(config, &Language::nodejs(), code, cwd, timeout).await } diff --git a/src/openhuman/runtime/pool/pool_tests.rs b/src/openhuman/runtime/pool/pool_tests.rs new file mode 100644 index 0000000000..75ef2f430b --- /dev/null +++ b/src/openhuman/runtime/pool/pool_tests.rs @@ -0,0 +1,83 @@ +//! Tests for the pooled-execution client. +//! +//! The pool's behaviour — warm workers, backpressure, recycling — belongs to the +//! `tinyruntime` module and is tested there. What is this core's decision is +//! whether a language pools by default, and how a module failure is classified, +//! because that classification is what keeps a job from running twice. + +use super::{PoolRunError, classify, node, python}; +use crate::openhuman::config::Config; +use crate::openhuman::modules::runtime::RuntimeCallError; + +#[test] +fn node_pools_by_default_and_python_does_not() { + // Not an oversight: a pooled Node job runs in its own worker thread with a + // fresh module graph, while a pooled Python job shares the interpreter with + // every other job on that worker. + let config = Config::default(); + assert!(node::enabled(&config.runtime_pool)); + assert!(!python::enabled(&config.runtime_pool)); +} + +#[test] +fn opting_python_in_turns_it_on() { + let mut config = Config::default(); + config.runtime_pool.python.enabled = Some(true); + assert!(python::enabled(&config.runtime_pool)); +} + +#[test] +fn turning_the_pool_off_wholesale_turns_it_off_for_every_language() { + let mut config = Config::default(); + config.runtime_pool.enabled = false; + config.runtime_pool.python.enabled = Some(true); + assert!(!node::enabled(&config.runtime_pool)); + assert!( + !python::enabled(&config.runtime_pool), + "an explicit per-language opt-in must not survive the master switch" + ); +} + +#[test] +fn a_saturated_pool_is_recognised_so_the_caller_does_not_spawn() { + // Falling back to a per-call spawn here would reintroduce exactly the + // resident memory the pool exists to cap. + let error = RuntimeCallError::Failed("the `nodejs` runtime pool is at capacity".to_string()); + assert!(matches!(classify(&error), PoolRunError::Saturated)); +} + +#[test] +fn a_post_dispatch_failure_is_recognised_so_the_job_is_not_re_run() { + // The distinction that matters most: this job may already have had its side + // effects, and a fallback spawn would repeat them. + let error = RuntimeCallError::Failed( + "the `nodejs` job failed after dispatch: the worker closed its protocol stream".to_string(), + ); + assert!(matches!(classify(&error), PoolRunError::PostDispatch(_))); +} + +#[test] +fn anything_else_is_pre_dispatch_so_the_caller_may_fall_back() { + // Including the module simply not being loaded: the job provably never ran, + // so the legacy per-call spawn is safe and is what keeps the tool working. + for error in [ + RuntimeCallError::Unavailable("no artifact for this host".to_string()), + RuntimeCallError::Failed("the `nodejs` job could not be dispatched: no worker".to_string()), + RuntimeCallError::InvalidRequest("no runtime provider for `ruby`".to_string()), + ] { + assert!( + matches!(classify(&error), PoolRunError::PreDispatch(_)), + "`{error}` was not classified as pre-dispatch" + ); + } +} + +#[test] +fn the_three_classifications_render_distinguishably() { + // They drive opposite caller behaviour, so their messages must not blur. + assert_eq!(PoolRunError::Saturated.to_string(), "runtime pool at capacity"); + let pre = PoolRunError::PreDispatch(anyhow::anyhow!("spawn failed")).to_string(); + assert!(pre.starts_with("pre-dispatch pool failure:"), "got {pre}"); + let post = PoolRunError::PostDispatch(anyhow::anyhow!("read wedged")).to_string(); + assert!(post.starts_with("post-dispatch pool failure:"), "got {post}"); +} diff --git a/src/openhuman/runtime/pool/python.rs b/src/openhuman/runtime/pool/python.rs index b7f3764f69..118e56bdcb 100644 --- a/src/openhuman/runtime/pool/python.rs +++ b/src/openhuman/runtime/pool/python.rs @@ -1,240 +1,38 @@ -//! Python pool backend: materialise the Python harness and submit inline jobs -//! to the shared [`LangPool`](super::pool::LangPool). -//! -//! Unlike the node backend, a job runs **in the worker's own interpreter** (no -//! per-job thread isolation — CPython can't safely kill a running thread), so a -//! Python job's soft deadline is enforced best-effort via `SIGALRM` on Unix and -//! otherwise falls back to the Rust-side hard deadline (which kills + respawns -//! the worker). `recycle_after_jobs` bounds cross-job module-state leakage. +//! Python pooled execution: whether this host wants it, and how to ask for it. -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::time::Duration; -use anyhow::Result; -use tokio::sync::OnceCell; +use tinyruntime_bus::Language; -use super::pool; -use super::types::{PoolExecOutcome, PoolLang, PoolSettings}; -use super::worker::WorkerLaunch; -use crate::openhuman::config::{RuntimePoolConfig, RuntimePoolLangConfig}; +use super::{PoolExecOutcome, PoolRunError}; +use crate::openhuman::config::{Config, RuntimePoolConfig}; -/// The bundled Python worker harness. -const WORKER_PY: &str = include_str!("pool_worker.py"); - -/// Written once per process (see [`super::ensure_worker_script`]). -static PYTHON_SCRIPT: OnceCell = OnceCell::const_new(); - -/// Whether inline `python` jobs should route through the pool. `runtime_python. -/// enabled` is already implied by the tool only being constructed when the -/// python runtime is enabled, so only the pool switches are checked here. +/// Whether inline `python` jobs should route through the pool. +/// +/// Python defaults **off**, and the asymmetry with Node is real rather than an +/// oversight. Jobs share one interpreter — CPython offers no worker-thread +/// equivalent and no safe way to kill a running thread — so reuse leaks +/// process-global state (`sys.modules`, `os.environ`, logging handlers, threads) +/// across otherwise unrelated runs. Opt in explicitly +/// (`[runtime_pool.python] enabled = true`) to accept that in exchange for the +/// warm-worker memory saving. +#[must_use] pub fn enabled(pool: &RuntimePoolConfig) -> bool { - // Python defaults OFF: jobs share one interpreter (no worker_thread - // equivalent), so reuse can leak process-global state (`sys.modules`, - // `os.environ`, logging handlers, threads) across otherwise-unrelated runs. - // Opt in explicitly (`[runtime_pool.python] enabled = true`) to accept that - // in exchange for the warm-worker memory win. pool.enabled && pool.python.is_enabled(false) } /// Run inline Python on a pooled, warm `python` worker. /// -/// `python_bin` / `bin_dir` come from the caller's already-resolved -/// [`ResolvedPython`](crate::openhuman::runtime::python::ResolvedPython). -/// `workspace_dir` and `lang_cfg` are injected at tool construction so this hot -/// path never re-reads config or re-writes the harness. `timeout` is the soft -/// per-job deadline (best-effort on Unix, hard-enforced by the Rust side). +/// # Errors +/// +/// [`PoolRunError`], classified so the caller knows whether falling back to a +/// per-call spawn is safe. pub async fn run_inline( - workspace_dir: &Path, - lang_cfg: &RuntimePoolLangConfig, - python_bin: &Path, - bin_dir: &Path, + config: &Config, code: String, cwd: Option, timeout: Option, -) -> Result { - let script = - super::ensure_worker_script(&PYTHON_SCRIPT, workspace_dir, "pool_worker.py", WORKER_PY) - .await - .map_err(super::pool::PoolRunError::PreDispatch)? - .to_string_lossy() - .into_owned(); - - let mut env = super::base_env(bin_dir); - // Line-buffered stdio so protocol frames flush promptly. - env.push(("PYTHONUNBUFFERED".to_string(), "1".to_string())); - - let launch = WorkerLaunch { - lang: PoolLang::Python, - bin: python_bin.to_path_buf(), - // `-u` unbuffered mirrors the runtime_python_server launch contract. - args: vec!["-u".to_string(), script], - env, - isolated_protocol: true, - }; - let settings = PoolSettings::from_lang_config(lang_cfg); - let pool = pool::ensure_pool(launch, settings).await; - - let cwd = cwd.map(|p| p.to_string_lossy().into_owned()); - pool.run_inline(code, cwd, timeout).await -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::openhuman::config::Config; - use crate::openhuman::runtime::pool::{all_stats, PoolLang}; - - /// Resolve the host `python3` binary + its bin dir, or `None` to skip. - fn system_python() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - for cmd in ["python3", "python"] { - if let Ok(out) = std::process::Command::new(cmd) - .args(["-c", "import sys; sys.stdout.write(sys.executable)"]) - .output() - { - if out.status.success() { - let bin = std::path::PathBuf::from(String::from_utf8_lossy(&out.stdout).trim()); - if let Some(dir) = bin.parent() { - return Some((bin.clone(), dir.to_path_buf())); - } - } - } - } - None - } - - async fn python_spawns() -> u64 { - all_stats() - .await - .into_iter() - .find(|(lang, _)| *lang == PoolLang::Python) - .map(|(_, stats)| stats.worker_spawns) - .unwrap_or(0) - } - - /// End-to-end: inline Python runs on the pool, reuses a warm worker, and - /// surfaces stdout / exit codes / cwd. Skips when no system python. - #[tokio::test] - async fn pooled_python_runs_inline_and_reuses_worker() { - let Some((python_bin, bin_dir)) = system_python() else { - eprintln!("[runtime_pool] test skipped: no system python on PATH"); - return; - }; - let tmp = std::env::temp_dir().join(format!("rt-pool-py-e2e-{}", std::process::id())); - std::fs::create_dir_all(&tmp).unwrap(); - - let mut config = Config::default(); - config.workspace_dir = tmp.clone(); - config.runtime_pool.python.max_workers = 1; - config.runtime_pool.python.recycle_after_jobs = 0; - let lang = config.runtime_pool.python.clone(); - - let spawns_before = python_spawns().await; - - let out1 = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "print(6 * 7)".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("py job 1 runs"); - assert!(out1.success(), "job 1 should succeed: {out1:?}"); - assert_eq!(out1.stdout.trim(), "42"); - - // Raising surfaces a non-zero exit + traceback on stderr. - let out2 = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "raise ValueError('boom')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("py job 2 runs"); - assert!(!out2.success(), "raising job should fail"); - assert!(out2.stderr.contains("boom"), "stderr was {:?}", out2.stderr); - - // cwd-relative read resolves against the job cwd. - std::fs::write(tmp.join("probe.txt"), "PY_REL_OK").unwrap(); - let out3 = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "print(open('./probe.txt').read(), end='')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("py job 3 runs"); - assert!(out3.success(), "cwd read should succeed: {out3:?}"); - assert_eq!(out3.stdout, "PY_REL_OK"); - - // User stdin is an isolated EOF stream, not the long-lived worker's - // NDJSON request pipe. - let stdin_out = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "import os, sys\nprint(repr(sys.stdin.read()))\nprint(os.read(0, 1))".to_string(), - Some(tmp.clone()), - Some(Duration::from_secs(2)), - ) - .await - .expect("stdin EOF job runs"); - assert!( - stdin_out.success(), - "python stdin should be EOF: {stdin_out:?}" - ); - assert_eq!(stdin_out.stdout, "''\nb''\n"); - - // A missing cwd must fail before executing user code. Continuing in the - // worker's inherited cwd would escape the requested action root. - let missing_cwd = tmp.join("deleted-action-root"); - let err = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "open('must-not-exist.txt', 'w').write('bad')".to_string(), - Some(missing_cwd), - None, - ) - .await - .expect_err("missing cwd must fail closed"); - assert!( - err.to_string().contains("failed to set worker cwd"), - "unexpected missing-cwd error: {err}" - ); - assert!(!tmp.join("must-not-exist.txt").exists()); - - // The harness-level error must not poison framing or worker reuse. - let out4 = run_inline( - &config.workspace_dir, - &lang, - &python_bin, - &bin_dir, - "print('AFTER_CWD_ERROR')".to_string(), - Some(tmp.clone()), - None, - ) - .await - .expect("job after cwd error runs"); - assert_eq!(out4.stdout, "AFTER_CWD_ERROR\n"); - - let spawns_after = python_spawns().await; - assert!( - spawns_after - spawns_before <= 1, - "expected warm-worker reuse: {} new spawns", - spawns_after - spawns_before - ); - - let _ = std::fs::remove_dir_all(&tmp); - } +) -> Result { + super::run_inline(config, &Language::python(), code, cwd, timeout).await } diff --git a/src/openhuman/runtime/pool/types_tests.rs b/src/openhuman/runtime/pool/types_tests.rs new file mode 100644 index 0000000000..da5c4b7991 --- /dev/null +++ b/src/openhuman/runtime/pool/types_tests.rs @@ -0,0 +1,83 @@ +//! Unit tests for the pooled-execution types. + +use std::time::Duration; + +use tinyruntime_bus::{ExecResponse, Language}; + +use super::{PoolExecOutcome, PoolLang, PoolSettings}; +use crate::openhuman::config::RuntimePoolLangConfig; + +fn outcome(exit_code: Option) -> PoolExecOutcome { + PoolExecOutcome { + stdout: String::new(), + stderr: String::new(), + exit_code, + timed_out: false, + elapsed: Duration::ZERO, + queue_wait: Duration::ZERO, + } +} + +#[test] +fn success_requires_a_clean_exit_and_no_timeout() { + assert!(outcome(Some(0)).success()); + assert!(outcome(None).success()); + assert!(!outcome(Some(1)).success()); + + let timed_out = PoolExecOutcome { + timed_out: true, + ..outcome(Some(0)) + }; + assert!(!timed_out.success(), "a job aborted at its deadline did not succeed"); +} + +#[test] +fn a_module_reply_keeps_run_time_and_queue_wait_apart() { + // A host that cannot tell a slow job from a busy pool will tune the wrong + // thing, so the two never collapse into one number. + let response = ExecResponse::new("out", "err", Some(0), "22.11.0").with_timings(12, 900); + let adapted = PoolExecOutcome::from_module(&response); + + assert_eq!(adapted.stdout, "out"); + assert_eq!(adapted.stderr, "err"); + assert_eq!(adapted.elapsed, Duration::from_millis(12)); + assert_eq!(adapted.queue_wait, Duration::from_millis(900)); + assert!(adapted.success()); +} + +#[test] +fn a_timed_out_reply_stays_timed_out_through_the_adaptation() { + let response = ExecResponse::new("", "", None, "22.11.0").with_timed_out(true); + let adapted = PoolExecOutcome::from_module(&response); + assert!(adapted.timed_out); + assert!(!adapted.success()); +} + +#[test] +fn each_pool_language_round_trips_through_its_bus_language() { + for lang in [PoolLang::Node, PoolLang::Python] { + assert_eq!(PoolLang::from_language(&lang.language()), Some(lang)); + } +} + +#[test] +fn an_unfamiliar_language_is_skipped_rather_than_guessed() { + // The module routes whatever its own configuration routes; a status surface + // should skip an entry this build has no pool concept for. + assert_eq!(PoolLang::from_language(&Language::new("ruby")), None); +} + +#[test] +fn settings_disable_idle_reaping_on_zero() { + let cfg = RuntimePoolLangConfig { + enabled: Some(true), + max_workers: 3, + idle_ttl_secs: 0, + recycle_after_jobs: 5, + max_queue_depth: 10, + }; + let settings = PoolSettings::from_lang_config(&cfg); + assert_eq!(settings.max_workers, 3); + assert!(settings.idle_ttl.is_none()); + assert_eq!(settings.recycle_after_jobs, 5); +} From 40644610055b2a0a9cff307e657a06f10bf68eed Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:11:02 +0300 Subject: [PATCH 17/55] refactor(ops): simplify bootstrap construction by passing root config Replace the explicit field cloning in NodeBootstrap and PythonBootstrap constructors with a direct reference to the root configuration, reducing code duplication and making the construction logic more concise. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/ops.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index bb0d8d51e7..a469177e56 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -124,11 +124,7 @@ pub fn all_tools_with_runtime( prefer_system = root_config.node.prefer_system, "[tools::ops] node runtime enabled — constructing shared NodeBootstrap" ); - Some(Arc::new(NodeBootstrap::new( - root_config.node.clone(), - action_dir.to_path_buf(), - reqwest::Client::new(), - ))) + Some(Arc::new(NodeBootstrap::new(Arc::clone(root_config)))) } else { tracing::debug!( "[tools::ops] node runtime disabled — shell PATH injection + node_exec/npm_exec suppressed" @@ -141,9 +137,7 @@ pub fn all_tools_with_runtime( prefer_system = root_config.runtime_python.prefer_system, "[tools::ops] python runtime enabled — constructing shared PythonBootstrap" ); - Some(Arc::new(PythonBootstrap::new( - root_config.runtime_python.clone(), - ))) + Some(Arc::new(PythonBootstrap::new(Arc::clone(root_config)))) } else { tracing::debug!( "[tools::ops] python runtime disabled — shell python/pip PATH injection suppressed" From ca1fa1ee7c7d9d069300ab135ebce433f0b2eacc Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:11:22 +0300 Subject: [PATCH 18/55] refactor(config): share a single Arc-wrapped Config across runtime bootstraps Replace multiple per-call Config clones with a single Arc-wrapped snapshot that is shared between the Node.js and Python runtime bootstraps. This ensures both language clients always see the same session configuration, preventing any possibility of version disagreement during tool execution. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/agent/harness_init/registry.rs | 10 +++------- src/openhuman/tools/ops.rs | 10 ++++++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/openhuman/agent/harness_init/registry.rs b/src/openhuman/agent/harness_init/registry.rs index 48ceb533b4..a90bbcf6f4 100644 --- a/src/openhuman/agent/harness_init/registry.rs +++ b/src/openhuman/agent/harness_init/registry.rs @@ -102,7 +102,7 @@ async fn python_is_done(config: &Config) -> bool { // `try_cached`), so an already-installed interpreter is detected without // entering a user-visible provisioning run (GH-5047). Never downloads. use crate::openhuman::runtime::python::PythonBootstrap; - PythonBootstrap::new(config.runtime_python.clone()) + PythonBootstrap::new(std::sync::Arc::new(config.clone())) .probe_installed() .await .is_some() @@ -113,7 +113,7 @@ async fn python_run(config: &Config) -> Result<(), String> { return Ok(()); } use crate::openhuman::runtime::python::PythonBootstrap; - PythonBootstrap::new(config.runtime_python.clone()) + PythonBootstrap::new(std::sync::Arc::new(config.clone())) .resolve() .await .map(|resolved| { @@ -249,11 +249,7 @@ fn node_runtime_step() -> HarnessInitStep { #[cfg(feature = "runtime-node")] fn build_node_bootstrap(config: &Config) -> crate::openhuman::runtime::node::NodeBootstrap { - crate::openhuman::runtime::node::NodeBootstrap::new( - config.node.clone(), - config.workspace_dir.clone(), - reqwest::Client::new(), - ) + crate::openhuman::runtime::node::NodeBootstrap::new(std::sync::Arc::new(config.clone())) } #[cfg(feature = "runtime-node")] diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index a469177e56..6678104181 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -109,6 +109,12 @@ pub fn all_tools_with_runtime( #[cfg(not(feature = "skills"))] let _ = (active_profile, skill_allowlist, profile_skills_root); + // One shared snapshot of this session's configuration for both language + // clients. They each hand it to the `tinyruntime` module on every call — + // the module holds no configuration of its own — so the two must not be + // able to disagree about which version this session asked for. + let shared_config = Arc::new(root_config.clone()); + // Build a session-scoped managed Node.js bootstrap once, so ShellTool, // NodeExecTool, and NpmExecTool all share the same memoised resolution // state. Disabled when `node.enabled = false` — in that case shell skips @@ -124,7 +130,7 @@ pub fn all_tools_with_runtime( prefer_system = root_config.node.prefer_system, "[tools::ops] node runtime enabled — constructing shared NodeBootstrap" ); - Some(Arc::new(NodeBootstrap::new(Arc::clone(root_config)))) + Some(Arc::new(NodeBootstrap::new(Arc::clone(&shared_config)))) } else { tracing::debug!( "[tools::ops] node runtime disabled — shell PATH injection + node_exec/npm_exec suppressed" @@ -137,7 +143,7 @@ pub fn all_tools_with_runtime( prefer_system = root_config.runtime_python.prefer_system, "[tools::ops] python runtime enabled — constructing shared PythonBootstrap" ); - Some(Arc::new(PythonBootstrap::new(Arc::clone(root_config)))) + Some(Arc::new(PythonBootstrap::new(Arc::clone(&shared_config)))) } else { tracing::debug!( "[tools::ops] python runtime disabled — shell python/pip PATH injection suppressed" From 5aa13f275c9cb8ab84119a14c68cdc9d25807c17 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:11:35 +0300 Subject: [PATCH 19/55] fix(tools): remove unused parameter from try_pool_inline The `resolved` parameter was removed from `try_pool_inline` because it is no longer needed; the function now passes the bootstrap configuration directly to `run_inline` instead of extracting individual fields from the resolved node. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/impl/system/node_exec.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/openhuman/tools/impl/system/node_exec.rs b/src/openhuman/tools/impl/system/node_exec.rs index 9604a32d33..38dafe41f5 100644 --- a/src/openhuman/tools/impl/system/node_exec.rs +++ b/src/openhuman/tools/impl/system/node_exec.rs @@ -402,7 +402,6 @@ impl NodeExecTool { async fn try_pool_inline( &self, code: &str, - resolved: &crate::openhuman::runtime::node::ResolvedNode, action_dir: &std::path::Path, timeout: Option, ) -> Option { @@ -418,10 +417,7 @@ impl NodeExecTool { return None; } match crate::openhuman::runtime::pool::node::run_inline( - &self.workspace_dir, - &self.pool_cfg.node, - &resolved.node_bin, - &resolved.bin_dir, + self.bootstrap.config(), code.to_string(), Some(action_dir.to_path_buf()), timeout, From 0a843cf4fc95dfefeba7b6ea2c188512ef82614f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:11:45 +0300 Subject: [PATCH 20/55] fix(tools): remove unused resolved parameter from pool inline calls The `resolved` parameter was being passed to `try_pool_inline` in both Node and Python executors but was not actually used by the underlying pool infrastructure. Removing it simplifies the call sites and eliminates a dead argument, making the code clearer and reducing the chance of confusion about which parameters are meaningful. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/impl/system/node_exec.rs | 2 +- src/openhuman/tools/impl/system/python_exec.rs | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/openhuman/tools/impl/system/node_exec.rs b/src/openhuman/tools/impl/system/node_exec.rs index 38dafe41f5..8e4592e2f8 100644 --- a/src/openhuman/tools/impl/system/node_exec.rs +++ b/src/openhuman/tools/impl/system/node_exec.rs @@ -304,7 +304,7 @@ impl NodeExecTool { // pool infrastructure failure also transparently falls back below. if let Some(code) = inline_code.as_deref() { if let Some(result) = self - .try_pool_inline(code, &resolved, &path_policy.action_dir, explicit_timeout) + .try_pool_inline(code, &path_policy.action_dir, explicit_timeout) .await { return Ok(result); diff --git a/src/openhuman/tools/impl/system/python_exec.rs b/src/openhuman/tools/impl/system/python_exec.rs index e0854b4fc6..d6d36c85af 100644 --- a/src/openhuman/tools/impl/system/python_exec.rs +++ b/src/openhuman/tools/impl/system/python_exec.rs @@ -364,10 +364,7 @@ impl PythonExecTool { return None; } match crate::openhuman::runtime::pool::python::run_inline( - &self.workspace_dir, - &self.pool_cfg.python, - &resolved.python_bin, - &resolved.bin_dir, + self.bootstrap.config(), code.to_string(), Some(action_dir.to_path_buf()), timeout, From 2fd130d4b1ad332369a7f4b5b8ab7499f8643049 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:11:53 +0300 Subject: [PATCH 21/55] chore(python_exec): remove unused parameter from try_pool_inline The `resolved` parameter was removed from the `try_pool_inline` method signature because it is no longer needed for the function's logic. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/impl/system/python_exec.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/openhuman/tools/impl/system/python_exec.rs b/src/openhuman/tools/impl/system/python_exec.rs index d6d36c85af..16d2a1e31c 100644 --- a/src/openhuman/tools/impl/system/python_exec.rs +++ b/src/openhuman/tools/impl/system/python_exec.rs @@ -356,7 +356,6 @@ impl PythonExecTool { async fn try_pool_inline( &self, code: &str, - resolved: &ResolvedPython, action_dir: &std::path::Path, timeout: Option, ) -> Option { From 6e5de96ae7e92b5df921727e2f291424ed4927c4 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:12:09 +0300 Subject: [PATCH 22/55] chore(runtime): clarify module docs after toolchain extraction Update the runtime module documentation to reflect that the toolchain download, extraction, and caching machinery has moved into the `tinyruntime` module, leaving the runtime directory as a client-side adapter. The python_exec tool call is also corrected to remove a stale reference to the resolved parameter that no longer exists in the function signature. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/mod.rs | 28 +++++++++++-------- .../tools/impl/system/python_exec.rs | 2 +- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/openhuman/runtime/mod.rs b/src/openhuman/runtime/mod.rs index 35ea265d37..86dae216eb 100644 --- a/src/openhuman/runtime/mod.rs +++ b/src/openhuman/runtime/mod.rs @@ -1,19 +1,25 @@ -//! Code-execution runtimes. +//! Code-execution runtimes: the client side. //! -//! The substrate agents, skills, and flows use to run untrusted-ish code: -//! managed Node and Python toolchains (download, extract, version-pin, invoke), -//! the long-lived worker pool in front of them, and the JavaScript evaluation -//! surface. +//! The substrate agents, skills, and flows use to run untrusted-ish code. What +//! is *here* is the client half — everything that downloads a toolchain, +//! verifies it, unpacks it, caches it, or keeps a warm worker in front of it now +//! lives in the `tinyruntime` module, behind +//! [`crate::openhuman::modules::runtime`]. //! -//! - [`node`] — managed Node toolchain + `node_exec` / `npm_exec` backing -//! - [`python`] — managed Python toolchain +//! That split is what this directory is for: adapting module answers onto the +//! types the rest of the core already names, so a migration of the machinery did +//! not become a migration of every caller. +//! +//! - [`node`] — the Node toolchain client, plus the ungated native-tool +//! bridge that shares its directory +//! - [`python`] — the Python interpreter client, plus stdio child launch //! - [`python_server`] — the persistent Python worker process -//! - [`pool`] — worker-pool lifecycle shared by the two runtimes +//! - [`pool`] — pooled execution, and the fallback classification //! - [`javascript`] — JavaScript evaluation surface //! -//! Family boundary == future gate boundary: `runtime-node` and `runtime-python` -//! are planned gates (`node` sheds the exclusive `xz2` + its static liblzma C -//! build). See `docs/specs/2026-08-02-core-kernel-domain-reorg.md`. +//! The archive and HTTP dependencies these modules used to carry went with the +//! machinery: a client that asks a module for a path needs neither a +//! decompressor nor a download pipeline. pub mod javascript; pub mod node; diff --git a/src/openhuman/tools/impl/system/python_exec.rs b/src/openhuman/tools/impl/system/python_exec.rs index 16d2a1e31c..0277a2f88a 100644 --- a/src/openhuman/tools/impl/system/python_exec.rs +++ b/src/openhuman/tools/impl/system/python_exec.rs @@ -265,7 +265,7 @@ impl PythonExecTool { // pool infrastructure failure also transparently falls back below. if let Some(code) = inline_code.as_deref() { if let Some(result) = self - .try_pool_inline(code, &resolved, &path_policy.action_dir, explicit_timeout) + .try_pool_inline(code, &path_policy.action_dir, explicit_timeout) .await { return Ok(result); From 046e7d4f348059a33d1301b211649c4c9bcbb569 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:15:07 +0300 Subject: [PATCH 23/55] refactor(runtime): switch bootstrap constructors to accept `Arc` Change the `NodeBootstrap` and `PythonBootstrap` constructors to take an `Arc` instead of a cloned sub-config, reducing allocations and simplifying the call sites. The node stub is also updated to hold the full config and expose a `config()` accessor, while the doc comments are cleaned up for clarity. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/stub.rs | 69 +++++++++++-------- .../runtime/python_server/kompress.rs | 2 +- src/openhuman/runtime/python_server/server.rs | 2 +- src/openhuman/runtime/python_server/spacy.rs | 2 +- src/openhuman/skills/runtime/ops.rs | 2 +- 5 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/openhuman/runtime/node/stub.rs b/src/openhuman/runtime/node/stub.rs index e7832d862b..5e6ff405cb 100644 --- a/src/openhuman/runtime/node/stub.rs +++ b/src/openhuman/runtime/node/stub.rs @@ -1,8 +1,7 @@ //! `runtime-node` disabled-build stub. //! //! Mirrors the *type* surface that always-compiled callers name, with no-op -//! behaviour. Only what is actually reached from outside the gate lives here — -//! the download/extract/resolve machinery is compiled out entirely. +//! behaviour. Only what is actually reached from outside the gate lives here. //! //! Why a stub rather than a leaf gate: [`NodeBootstrap`] appears in the **field //! type** of `tools::impl::system::ShellTool` (`Option>`, for @@ -12,23 +11,23 @@ //! `npm_exec`) are leaf-gated at their call sites instead, because registration //! sites want absence. //! -//! Off-state: `try_cached` / `probe_installed` return `None`, so the shell -//! simply never prepends a managed `bin/` dir — the same path taken today when -//! `node.enabled = false`. `resolve()` is the one erroring method and is only -//! reachable from `harness_init`'s bootstrap step, itself gated off; it returns -//! a build fact so a stray caller reports something actionable. +//! Off-state: `try_cached` and `probe_installed` return `None`, so the shell +//! simply never prepends a managed `bin/` directory — the same path taken when +//! `node.enabled = false`. `resolve` is the one erroring method and returns a +//! build fact so a stray caller reports something actionable. //! -//! Note: this stub carries **only** the managed-Node toolchain type surface. +//! Note: this stub carries **only** the Node toolchain type surface. //! [`super::ops`] and [`super::types`] are not stubbed — they are the generic //! native-tool dispatcher and its inert serde types, always compiled so the -//! ungated `flows` `NativeToolBackend` can keep dispatching `oh:*` tools when -//! the managed Node runtime is off. +//! ungated `flows` backend can keep dispatching `oh:*` tools when the managed +//! Node runtime is off. use std::path::PathBuf; +use std::sync::Arc; -use anyhow::Result; +use anyhow::{Result, anyhow}; -use crate::openhuman::config::schema::NodeConfig; +use crate::openhuman::config::Config; /// Returned by [`NodeBootstrap::resolve`] in a `runtime-node`-less build. /// Phrased as a build fact, matching the `mcp` / `tui` CLI-arm convention. @@ -40,50 +39,66 @@ pub const RUNTIME_NODE_DISABLED_MESSAGE: &str = /// arms and imports still resolve. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum NodeSource { + /// Reused a compatible `node` already on the host. System, + /// A managed distribution. Managed, } /// Fully-resolved Node toolchain. Never constructed in a disabled build. #[derive(Debug, Clone)] pub struct ResolvedNode { + /// Directory to prepend to `PATH`. pub bin_dir: PathBuf, + /// Absolute path to the `node` binary. pub node_bin: PathBuf, + /// Absolute path to the `npm` launcher. pub npm_bin: PathBuf, + /// Version string without the leading `v`. pub version: String, + /// Where the toolchain came from. pub source: NodeSource, } -/// Disabled-build bootstrap: constructs, resolves to nothing. +/// Inert stand-in for the toolchain client. #[derive(Debug)] pub struct NodeBootstrap { - _config: NodeConfig, - _workspace_dir: PathBuf, + config: Arc, } impl NodeBootstrap { - /// Signature-compatible with the real constructor. The `reqwest::Client` is - /// accepted and dropped — a disabled build never downloads. - pub fn new(config: NodeConfig, workspace_dir: PathBuf, _client: reqwest::Client) -> Self { - Self { - _config: config, - _workspace_dir: workspace_dir, - } + /// Build a stub over this host's configuration. + /// + /// Takes the same argument as the real client so construction sites do not + /// need their own gate. + #[must_use] + pub fn new(config: Arc) -> Self { + Self { config } } - /// Always `None` — nothing is cached because nothing resolves. + /// The configuration this bootstrap would resolve under. + #[must_use] + pub fn config(&self) -> &Config { + &self.config + } + + /// Always `None`: nothing resolves in a disabled build. + #[must_use] pub fn try_cached(&self) -> Option { None } - /// Always `None`. The real implementation probes the on-disk install; there - /// is no install path in a disabled build. + /// Always `None`: nothing is provisioned in a disabled build. pub async fn probe_installed(&self) -> Option { None } - /// Always `Err`. See [`RUNTIME_NODE_DISABLED_MESSAGE`]. + /// Always an error naming the missing feature. + /// + /// # Errors + /// + /// Always, with [`RUNTIME_NODE_DISABLED_MESSAGE`]. pub async fn resolve(&self) -> Result { - anyhow::bail!(RUNTIME_NODE_DISABLED_MESSAGE) + Err(anyhow!(RUNTIME_NODE_DISABLED_MESSAGE)) } } diff --git a/src/openhuman/runtime/python_server/kompress.rs b/src/openhuman/runtime/python_server/kompress.rs index ead0e22967..ec3bed58f3 100644 --- a/src/openhuman/runtime/python_server/kompress.rs +++ b/src/openhuman/runtime/python_server/kompress.rs @@ -130,7 +130,7 @@ pub async fn ensure_kompress(config: &Config) -> Result { config.tokenjuice.ml_model_id ); - let base = PythonBootstrap::new(config.runtime_python.clone()) + let base = PythonBootstrap::new(std::sync::Arc::new(config.clone())) .resolve() .await .context("resolving base python for kompress venv")?; diff --git a/src/openhuman/runtime/python_server/server.rs b/src/openhuman/runtime/python_server/server.rs index aaf4742b34..b0792b7931 100644 --- a/src/openhuman/runtime/python_server/server.rs +++ b/src/openhuman/runtime/python_server/server.rs @@ -403,7 +403,7 @@ async fn prepare_launch(config: &Config) -> Result { push_kompress_env(&mut env, config, &rt.hf_home); rt.python_bin } else { - crate::openhuman::runtime::python::PythonBootstrap::new(config.runtime_python.clone()) + crate::openhuman::runtime::python::PythonBootstrap::new(std::sync::Arc::new(config.clone())) .resolve() .await? .python_bin diff --git a/src/openhuman/runtime/python_server/spacy.rs b/src/openhuman/runtime/python_server/spacy.rs index 8d0db1b365..17c7f7cd62 100644 --- a/src/openhuman/runtime/python_server/spacy.rs +++ b/src/openhuman/runtime/python_server/spacy.rs @@ -101,7 +101,7 @@ pub async fn ensure_spacy(config: &Config) -> Result { SPACY_MODEL ); - let bootstrap = PythonBootstrap::new(config.runtime_python.clone()); + let bootstrap = PythonBootstrap::new(std::sync::Arc::new(config.clone())); let base = bootstrap .resolve() .await diff --git a/src/openhuman/skills/runtime/ops.rs b/src/openhuman/skills/runtime/ops.rs index a901e7beac..e20b999bc0 100644 --- a/src/openhuman/skills/runtime/ops.rs +++ b/src/openhuman/skills/runtime/ops.rs @@ -132,7 +132,7 @@ async fn resolve_python(config: &Config) -> ResolvedRuntimeSummary { error: Some("python runtime disabled".to_string()), }; } - let bootstrap = PythonBootstrap::new(config.runtime_python.clone()); + let bootstrap = PythonBootstrap::new(std::sync::Arc::new(config.clone())); match bootstrap.resolve().await { Ok(resolved) => ResolvedRuntimeSummary { runtime: "python".to_string(), From a6e38a30f2f4fe45362032d142a68a86c51e9e9f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:22:33 +0300 Subject: [PATCH 24/55] fix(runtime): simplify NodeBootstrap construction Changed the `resolve_node` function to pass a single `Arc` to `NodeBootstrap::new` instead of cloning individual fields, reducing unnecessary cloning and simplifying the constructor call. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/skills/runtime/ops.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/openhuman/skills/runtime/ops.rs b/src/openhuman/skills/runtime/ops.rs index e20b999bc0..e232ad8f5f 100644 --- a/src/openhuman/skills/runtime/ops.rs +++ b/src/openhuman/skills/runtime/ops.rs @@ -84,11 +84,7 @@ async fn resolve_node(config: &Config) -> ResolvedRuntimeSummary { error: Some("node runtime disabled".to_string()), }; } - let bootstrap = NodeBootstrap::new( - config.node.clone(), - config.workspace_dir.clone(), - reqwest::Client::new(), - ); + let bootstrap = NodeBootstrap::new(std::sync::Arc::new(config.clone())); match bootstrap.resolve().await { Ok(resolved) => ResolvedRuntimeSummary { runtime: "node".to_string(), From 311907492f034c1ddc4f0b3fd73919c4ae2a922c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:27:23 +0300 Subject: [PATCH 25/55] docs(runtime): update module-level doc comment and narrow re-exports The module documentation was rewritten to reflect the current architecture, where the Node.js backend is now a client of the `tinyruntime` module rather than a direct implementation. The re-exports under the `runtime-node` feature were reduced to only `execute_tool` and `list_tools`, removing several items that are no longer needed at this facade layer. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/javascript/mod.rs | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/src/openhuman/runtime/javascript/mod.rs b/src/openhuman/runtime/javascript/mod.rs index f3b2be0867..5932f89ac5 100644 --- a/src/openhuman/runtime/javascript/mod.rs +++ b/src/openhuman/runtime/javascript/mod.rs @@ -1,18 +1,17 @@ //! First-class JavaScript runtime surface. //! -//! Today the implementation backend is the managed Node.js runtime in -//! [`crate::openhuman::runtime::node`]. This module exists so the rest of the -//! core talks to a language slot (`javascript`) rather than directly to a -//! specific backend. That keeps the door open for future sibling modules like -//! `python`, `ruby`, or a different JavaScript backend. - +//! The implementation backend is the managed Node.js toolchain in +//! [`crate::openhuman::runtime::node`], which is itself a client for the +//! `tinyruntime` module. This facade exists so the rest of the core talks to a +//! language slot (`javascript`) rather than to a specific backend — which is +//! also how the module underneath was swapped without the callers noticing. +//! //! ## Gating (`runtime-node`) //! -//! The facade itself is always compiled — `ShellTool` imports `NodeBootstrap` +//! The facade itself is always compiled — `ShellTool` imports [`NodeBootstrap`] //! through it — but the re-exports split. The bootstrap type surface comes from -//! `node`'s stub when the feature is off; the download/extract/dispatch -//! machinery and the controller pair are gated, because their only consumers -//! are themselves gated off. +//! `node`'s stub when the feature is off; the dispatch machinery and the +//! controller pair are gated, because their only consumers are gated off too. pub use crate::openhuman::runtime::node::{ ExecuteToolOutcome, NodeBootstrap, NodeSource, ResolvedNode, @@ -26,7 +25,4 @@ pub use crate::openhuman::runtime::node::{ all_runtime_node_registered_controllers as all_javascript_registered_controllers, }; #[cfg(feature = "runtime-node")] -pub use crate::openhuman::runtime::node::{ - atomic_install, detect_system_node, download_distribution, execute_tool, extract_distribution, - fetch_shasums, list_tools, parse_node_version, NodeDistribution, SystemNode, -}; +pub use crate::openhuman::runtime::node::{execute_tool, list_tools}; From e56ed8ba30155f68bc42eb49903bfb0501bf1283 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:41:55 +0300 Subject: [PATCH 26/55] refactor(tools): simplify test bootstrap construction Replace the verbose manual construction of NodeBootstrap in test code with a single call to `Config::default()`, reducing duplication and making the tests easier to maintain. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/impl/system/node_exec.rs | 8 +++----- src/openhuman/tools/impl/system/npm_exec.rs | 16 ++++++---------- src/openhuman/tools/impl/system/shell.rs | 13 +++---------- 3 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/openhuman/tools/impl/system/node_exec.rs b/src/openhuman/tools/impl/system/node_exec.rs index 8e4592e2f8..849c506436 100644 --- a/src/openhuman/tools/impl/system/node_exec.rs +++ b/src/openhuman/tools/impl/system/node_exec.rs @@ -859,11 +859,9 @@ mod tests { }), ..SecurityPolicy::default() }); - let bootstrap = Arc::new(NodeBootstrap::new( - NodeConfig::default(), - temp.path().to_path_buf(), - reqwest::Client::new(), - )); + let bootstrap = Arc::new(NodeBootstrap::new(Arc::new( + crate::openhuman::config::Config::default(), + ))); let tool = NodeExecTool::new( security, Arc::new(NativeRuntime::new()), diff --git a/src/openhuman/tools/impl/system/npm_exec.rs b/src/openhuman/tools/impl/system/npm_exec.rs index b511df668f..9f1225fb82 100644 --- a/src/openhuman/tools/impl/system/npm_exec.rs +++ b/src/openhuman/tools/impl/system/npm_exec.rs @@ -656,11 +656,9 @@ mod tests { }), ..SecurityPolicy::default() }); - let bootstrap = Arc::new(NodeBootstrap::new( - NodeConfig::default(), - temp.path().to_path_buf(), - reqwest::Client::new(), - )); + let bootstrap = Arc::new(NodeBootstrap::new(Arc::new( + crate::openhuman::config::Config::default(), + ))); let tool = NpmExecTool::new(security, Arc::new(NativeRuntime::new()), bootstrap); let result = tool @@ -702,11 +700,9 @@ mod tests { }), ..SecurityPolicy::default() }); - let bootstrap = Arc::new(NodeBootstrap::new( - NodeConfig::default(), - temp.path().to_path_buf(), - reqwest::Client::new(), - )); + let bootstrap = Arc::new(NodeBootstrap::new(Arc::new( + crate::openhuman::config::Config::default(), + ))); let tool = NpmExecTool::new(security, Arc::new(NativeRuntime::new()), bootstrap); let result = tool diff --git a/src/openhuman/tools/impl/system/shell.rs b/src/openhuman/tools/impl/system/shell.rs index 6af1c21ed8..ee31aa18f1 100644 --- a/src/openhuman/tools/impl/system/shell.rs +++ b/src/openhuman/tools/impl/system/shell.rs @@ -1478,16 +1478,9 @@ mod tests { /// self-resolved in `runtime_path_for_command` — see the python branch.) #[tokio::test] async fn shell_does_not_resolve_or_install_node_on_its_own() { - let node = Arc::new(NodeBootstrap::new( - crate::openhuman::config::schema::NodeConfig { - enabled: true, - version: "v22.11.0".to_string(), - cache_dir: String::new(), - prefer_system: true, - }, - std::env::temp_dir(), - reqwest::Client::new(), - )); + let node = Arc::new(NodeBootstrap::new(Arc::new( + crate::openhuman::config::Config::default(), + ))); let tool = ShellTool::with_language_bootstraps( test_security(AutonomyLevel::Full), test_runtime(), From 02815849c8ce852409d3e5637ceee0407a6ba63b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:49:26 +0300 Subject: [PATCH 27/55] chore(exec): remove unused NodeConfig import from test modules Removed the unused `NodeConfig` import from test functions in the node and npm exec modules, and replaced the `ResolvedPython` import with just `PythonBootstrap` in the python exec module to clean up unused dependencies. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/impl/system/node_exec.rs | 1 - src/openhuman/tools/impl/system/npm_exec.rs | 2 -- src/openhuman/tools/impl/system/python_exec.rs | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/openhuman/tools/impl/system/node_exec.rs b/src/openhuman/tools/impl/system/node_exec.rs index 849c506436..d4d85a3136 100644 --- a/src/openhuman/tools/impl/system/node_exec.rs +++ b/src/openhuman/tools/impl/system/node_exec.rs @@ -839,7 +839,6 @@ mod tests { #[tokio::test] async fn inline_code_cannot_write_to_sibling_profile() { use crate::openhuman::agent::host_runtime::NativeRuntime; - use crate::openhuman::config::schema::NodeConfig; use crate::openhuman::security::policy::ActiveProfileGuard; use crate::openhuman::security::AutonomyLevel; diff --git a/src/openhuman/tools/impl/system/npm_exec.rs b/src/openhuman/tools/impl/system/npm_exec.rs index 9f1225fb82..2445a43fc8 100644 --- a/src/openhuman/tools/impl/system/npm_exec.rs +++ b/src/openhuman/tools/impl/system/npm_exec.rs @@ -636,7 +636,6 @@ mod tests { #[tokio::test] async fn args_cannot_target_sibling_profile() { use crate::openhuman::agent::host_runtime::NativeRuntime; - use crate::openhuman::config::schema::NodeConfig; use crate::openhuman::security::policy::ActiveProfileGuard; use crate::openhuman::security::AutonomyLevel; @@ -677,7 +676,6 @@ mod tests { #[tokio::test] async fn symlinked_cwd_cannot_target_sibling_profile() { use crate::openhuman::agent::host_runtime::NativeRuntime; - use crate::openhuman::config::schema::NodeConfig; use crate::openhuman::security::policy::ActiveProfileGuard; use crate::openhuman::security::AutonomyLevel; use std::os::unix::fs::symlink; diff --git a/src/openhuman/tools/impl/system/python_exec.rs b/src/openhuman/tools/impl/system/python_exec.rs index 0277a2f88a..f5203548a6 100644 --- a/src/openhuman/tools/impl/system/python_exec.rs +++ b/src/openhuman/tools/impl/system/python_exec.rs @@ -17,7 +17,7 @@ //! paths and sandboxed runs always use the per-call spawn. use crate::openhuman::agent::host_runtime::RuntimeAdapter; -use crate::openhuman::runtime::python::{PythonBootstrap, ResolvedPython}; +use crate::openhuman::runtime::python::PythonBootstrap; use crate::openhuman::security::{CommandClass, GateDecision, SecurityPolicy}; use crate::openhuman::tools::traits::{ PermissionLevel, Tool, ToolCallOptions, ToolResult, ToolTimeout, From a6ebc7f26bdea577dd6a4ce1158c5514b0dff964 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:50:15 +0300 Subject: [PATCH 28/55] chore(deps): remove xz2 dependency from runtime-node feature The xz2 crate with its static liblzma C build has been removed from the manifest, as toolchain archive extraction now lives in the tinyruntime module. The runtime-node feature gate is kept as an empty list to continue controlling the presence of the Node.js tools and controllers themselves. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.toml | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e0bc084590..9a536aca0a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -354,18 +354,18 @@ sha2 = "0.10" # graph is a hard cargo error, not a warning. Test code that must inspect a # ledger goes through `tinycortex::git2`. hmac = "0.12" -# Archive extraction for the Node.js runtime bootstrap. Unix Node -# distributions ship as .tar.xz, Windows as .zip. `xz2` with `static` -# bundles liblzma so we don't need it as a system dependency. +# Archive handling for the Piper voice installer and the document tools. The +# Node.js and Python toolchain archives are no longer unpacked here — the +# `tinyruntime` module owns that — which is what let `xz2` and its static +# liblzma C build leave this manifest entirely. tar = "0.4" -xz2 = { version = "0.1", features = ["static"], optional = true } zip = { version = "2", default-features = false, features = ["deflate"] } # gzip decoder for the Piper tar.gz binary releases on macOS / Linux. Already # pulled in transitively by zip's `deflate` feature; declared directly so # the installer module can `use flate2::read::GzDecoder`. flate2 = "1" -# Real timeout for `node --version` probes in the runtime resolver. Guards -# against a broken shim on PATH hanging the bootstrap forever. +# Real timeout around a blocking child wait, for the Claude Code auth probe. +# Guards against a broken binary on PATH hanging the caller forever. wait-timeout = "0.2" uuid = { version = "1", features = ["v4"] } anyhow = "1.0" @@ -645,7 +645,7 @@ proptest = "1" # web3's ethers/secp256k1 cohort, `documents`' zstd/bzip2 native builds, # `voice`+`inference`'s cpal/lettre/arboard/enigo/rdev stack, `contacts`' # macOS objc2 cohort, `crash-reporting`'s sentry tree, and `tui`'s -# ratatui/crossterm, and `runtime-node`'s xz2/liblzma. Turning them off takes a +# ratatui/crossterm. Turning them off takes a # bare `cargo check` from 540 packages / 7 native builds down to ~350 / 2, which # is the inner loop every contributor pays on every edit. # @@ -847,13 +847,17 @@ web3 = [ "modules", ] -# Managed Node.js runtime: `runtime::node` (download / verify / extract / install -# a pinned toolchain), the `runtime::javascript` language slot over it, -# `runtime::pool::node`, the `node_exec` / `npm_exec` agent tools, and the -# `node_runtime` harness-init step. Default-ON. Slim builds opt out via -# `--no-default-features --features ""`, which drops -# the exclusive `xz2` AND its static liblzma C build — the FIRST native -# toolchain build this gating program removes (6 native -> 5). +# Managed Node.js runtime: `runtime::node` (the client that asks the +# `tinyruntime` module for a toolchain), the `runtime::javascript` language slot +# over it, `runtime::pool::node`, the `node_exec` / `npm_exec` agent tools, and +# the `node_runtime` harness-init step. Default-ON. Slim builds opt out via +# `--no-default-features --features ""`. +# +# This gate no longer sheds a dependency. It used to carry the exclusive `xz2` +# and its static liblzma C build; downloading and unpacking a toolchain moved to +# the `tinyruntime` module, so that native build left this manifest for every +# configuration rather than only for slim ones. What the gate still buys is the +# absence of the tools and controllers themselves. # # FACADE + STUB, not a leaf gate. `ShellTool` holds `Option>` # for managed-Node PATH injection and `tools/impl/system/shell.rs` is kernel, so @@ -869,9 +873,7 @@ web3 = [ # # Off-state: `try_cached`/`probe_installed` return `None`, so the shell never # prepends a managed bin dir — identical to today's `node.enabled = false` path. -# `tar` and `zip` are NOT shed: `tar` is shared with `inference` (install_piper) -# and `runtime::python`, `zip` with `inference` and the document tools. -runtime-node = ["dep:xz2"] +runtime-node = [] # macOS Contacts seeding for the people domain: `memory::people::address_book` # reads CNContactStore to seed handles. Default-ON. Slim / headless builds opt From 2693c6f1a51b3027a7205d96e6e3a17fa8f05643 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 20:54:20 +0300 Subject: [PATCH 29/55] chore(deps): remove unused xz2 and lzma-sys dependencies The xz2 crate and its transitive dependency lzma-sys have been removed from the lock file, as they are no longer required by any direct dependency in the project. Auto-committed-on: dragonfly Co-authored-by: Medulla --- Cargo.lock | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 152ba48649..c2a011c67b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3387,17 +3387,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac_address" version = "1.1.8" @@ -4221,7 +4210,6 @@ dependencies = [ "windows-sys 0.61.2", "wiremock", "x25519-dalek", - "xz2", "zeroize", "zip", ] @@ -8538,15 +8526,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.7.5" From b7c705ef446d5240c088bf63ed9409f416e8a9ae Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:13:07 +0300 Subject: [PATCH 30/55] docs(runtime): document extraction of toolchain install logic into tinyruntime module Update the Node.js and Python runtime READMEs to reflect the architectural change that moved all toolchain installation, download, verification, and extraction logic into the shared `tinyruntime` module. The Node runtime now acts as a thin client that delegates to the module and adapts its response, while the Python runtime similarly sheds its install pipeline and retains only the process-launch helper for long-lived children. Both READMEs clarify the new separation of concerns, updated file listings, and the rationale for keeping local memoisation despite the module's own caching. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/README.md | 156 +++++++++++++++---------- src/openhuman/runtime/python/README.md | 108 ++++++++--------- 2 files changed, 151 insertions(+), 113 deletions(-) diff --git a/src/openhuman/runtime/node/README.md b/src/openhuman/runtime/node/README.md index ba3d97de17..9ac7da526c 100644 --- a/src/openhuman/runtime/node/README.md +++ b/src/openhuman/runtime/node/README.md @@ -1,102 +1,140 @@ -# runtime_node +# runtime/node -Managed **Node.js runtime** for the core, plus a thin **tool bridge** that lists and dispatches agent-callable tools through a `javascript` RPC namespace. The runtime half resolves or installs a pinned Node.js toolchain (reusing a compatible host `node` when present, otherwise downloading + SHA-256-verifying + extracting an official distribution from nodejs.org) so that `node_exec` / `npm_exec` / `shell` tools and Node-dependent skills have a trusted `node`/`npm` on a stable path. The bridge half exposes the full agent tool registry over JSON-RPC under `javascript.*` so callers (e.g. an embedded JS host) can enumerate and run tools by name. The public-facing language slot is the sibling [`javascript`](../javascript/) module, which re-exports this module's surface under `javascript`-prefixed names. +Two unrelated things share this directory, and the difference is the first thing +to understand about it. + +**The toolchain client** asks the `tinyruntime` module for a Node.js toolchain +and adapts the answer onto `ResolvedNode`, so `node_exec`, `npm_exec`, `shell`, +and Node-dependent skills have a trusted `node`/`npm` on a stable path. + +**The tool bridge** exposes the full agent tool registry over JSON-RPC under +`javascript.*`, so an embedded JS host can enumerate and run tools by name. It +has nothing to do with Node beyond being reachable from JavaScript, which is why +it is not gated with the rest. + +The public-facing language slot is the sibling [`javascript`](../javascript/) +module, which re-exports this module's surface under `javascript`-prefixed names. + +## What moved out + +Everything that used to make this the largest module in `runtime/`: system-node +probing, distribution selection, `SHASUMS256.txt` fetching, streaming download +with SHA-256 verification, `.tar.xz` / `.zip` extraction, atomic install, cache +roots, and the guards against a workspace-vendored fake install tree. + +That is all in the `tinyruntime` module now, where one implementation serves +every language, and it is reached through +[`modules::runtime`](../../modules/runtime.rs). The visible consequence for this +repository is that `xz2` and its static liblzma C build left the manifest +entirely — the first native toolchain build removed rather than merely gated. ## Responsibilities -- Detect a compatible system `node` on `PATH` (major-version match) and verify `npm` is also usable before reusing it. -- Resolve/install a managed Node.js toolchain when no compatible system node exists: pick the host archive, fetch `SHASUMS256.txt`, download with streaming SHA-256 verification, extract (`.tar.xz` / `.zip`), and atomically install into a user-owned cache root. -- Memoise the resolved toolchain behind a `tokio::sync::Mutex` so concurrent callers never race the download/extract/install pipeline; offer a non-blocking `try_cached()` peek for transparent `PATH` injection. -- Build the full agent tool registry on demand and expose two RPC controllers: list tool metadata, and execute a named tool returning an MCP-style `ToolResult`. -- Publish `ToolExecutionStarted` / `ToolExecutionCompleted` domain events around bridge tool execution. +- Ask the module to resolve a Node toolchain, installing one when the host has + none, and adapt the reply onto `ResolvedNode` (`node_bin`, `npm_bin`, + `bin_dir`, `version`, `source`). +- Memoise that answer locally so `try_cached()` can answer **without awaiting** — + the shell consults it on every command to decide whether to prepend a managed + `bin/` directory to `PATH`, and a blocking call there would make every + unrelated command wait on a bus round trip. +- Build the full agent tool registry on demand and expose two RPC controllers: + list tool metadata, and execute a named tool returning an MCP-style + `ToolResult`. +- Publish `ToolExecutionStarted` / `ToolExecutionCompleted` around bridge tool + execution. ## Key files | File | Role | | --- | --- | -| `src/openhuman/runtime/node/mod.rs` | Export-focused: submodule decls + `pub use` re-exports, including `all_runtime_node_controller_schemas` / `all_runtime_node_registered_controllers`. | -| `src/openhuman/runtime/node/resolver.rs` | Synchronous system-node probe. `detect_system_node`, `parse_node_version`, `SystemNode`. `PATH` walk with execute-bit filtering, `node --version` / `npm --version` probes with a 5s timeout. Major-version match only. | -| `src/openhuman/runtime/node/bootstrap.rs` | Orchestrator. `NodeBootstrap` (serialised + memoised `resolve()`, `try_cached()`), `ResolvedNode`, `NodeSource`. Picks system vs managed, computes the cache root (user cache by default, never workspace-local unless forced), guards against cache-root escape / spoofed installs via canonicalised `starts_with`. | -| `src/openhuman/runtime/node/downloader.rs` | `NodeDistribution` (host triple → archive name/URL), `fetch_shasums`, `download_distribution`. Streams to disk while hashing; **mandatory** SHA-256 match or the partial file is deleted. | -| `src/openhuman/runtime/node/extractor.rs` | `extract_distribution` (`.tar.xz` via `xz2`+`tar`, `.zip` via `zip`, both in `spawn_blocking`), `atomic_install` (rename into place with backup/restore). Asserts a single top-level folder per archive. | -| `src/openhuman/runtime/node/ops.rs` | Bridge logic: `build_runtime_tools` (assembles `SecurityPolicy`, audit logger, `NativeRuntime`, `Memory`, then `tools::all_tools_with_runtime`), `list_tools`, `execute_tool` (event publish + timing). | -| `src/openhuman/runtime/node/rpc.rs` | RPC param structs (`ListToolsParams`, `ExecuteToolParams`) and `*_handler` fns; loads config via `config::rpc` and delegates through the `javascript` alias, wrapping results in `RpcOutcome`. | -| `src/openhuman/runtime/node/schemas.rs` | Controller schemas + registered controllers for `javascript_list_tools` / `javascript_execute_tool`; `handle_*` deserialise params and call `rpc.rs`. | -| `src/openhuman/runtime/node/types.rs` | `RuntimeToolSummary`, `ExecuteToolOutcome` serde types. | +| `mod.rs` | Export-focused: submodule decls, the `runtime-node` gate, and `pub use` re-exports including the controller registry pair. | +| `bootstrap.rs` | The toolchain client. `NodeBootstrap` (`resolve`, `probe_installed`, `try_cached`), `ResolvedNode`, `NodeSource`. Adapts a module `ResolvedRuntime`; derives `npm` when the provider does not report it. | +| `stub.rs` | Type surface for `runtime-node`-less builds. `try_cached`/`probe_installed` return `None`; `resolve` errors with a build fact. | +| `ops.rs` | Bridge logic: `build_runtime_tools`, `list_tools`, `execute_tool` (event publish + timing). | +| `rpc.rs` | RPC param structs and `*_handler` fns; loads config and delegates through the `javascript` alias. | +| `schemas.rs` | Controller schemas + registered controllers for `javascript_list_tools` / `javascript_execute_tool`. | +| `types.rs` | `RuntimeToolSummary`, `ExecuteToolOutcome` serde types. | ## Public surface -From `mod.rs` re-exports: - -- Bootstrap: `NodeBootstrap`, `NodeSource`, `ResolvedNode`. -- Downloader: `download_distribution`, `fetch_shasums`, `NodeDistribution`. -- Extractor: `atomic_install`, `extract_distribution`. -- Resolver: `detect_system_node`, `parse_node_version`, `SystemNode`. +- Client: `NodeBootstrap`, `NodeSource`, `ResolvedNode`. - Bridge ops: `execute_tool`, `list_tools`. - Types: `RuntimeToolSummary`, `ExecuteToolOutcome` (via `types`). -- Controller registry pair: `all_runtime_node_controller_schemas`, `all_runtime_node_registered_controllers`. +- Controller registry pair: `all_runtime_node_controller_schemas`, + `all_runtime_node_registered_controllers`. ## RPC / controllers -Registered under namespace `javascript` (schemas wired into `src/core/all.rs` via the `javascript` module's `all_javascript_*` aliases, not under a `runtime_node` name): +Registered under namespace `javascript` (schemas wired into `src/core/all.rs` +via the `javascript` module's `all_javascript_*` aliases, not under a +`runtime_node` name): | Method | Inputs | Output | | --- | --- | --- | -| `javascript.list_tools` | none | `tools`: array of tool metadata (name, description, category, permission_level, scope, supports_markdown, parameters). | -| `javascript.execute_tool` | `tool_name` (required), `args` (optional, defaults `{}`), `prefer_markdown` (optional bool) | `tool_name`, `elapsed_ms`, `result` (MCP-style `ToolResult`: `{content, is_error, markdownFormatted?}`). | +| `javascript.list_tools` | none | `tools`: array of tool metadata. | +| `javascript.execute_tool` | `tool_name` (required), `args` (optional, defaults `{}`), `prefer_markdown` (optional bool) | `tool_name`, `elapsed_ms`, `result`. | -Handlers load config via `config::rpc::load_config_with_timeout`, return `RpcOutcome` (`into_cli_compatible_json`). Unknown tool name → error `unknown tool \`\``. +Unknown tool name → error ``unknown tool `` ``. ## Agent tools -This module owns **no** tools of its own. Instead it builds the *entire* agent tool registry on demand (`tools::all_tools_with_runtime`) to back the `javascript.execute_tool` / `javascript.list_tools` bridge. The actual `node_exec`, `npm_exec`, and `shell` tools live in `src/openhuman/tools/impl/system/` and consume this module's `NodeBootstrap` for binary resolution / `PATH` injection. +This module owns **no** tools of its own. It builds the *entire* agent tool +registry on demand (`tools::all_tools_with_runtime`) to back the bridge. The +`node_exec`, `npm_exec`, and `shell` tools live in +`src/openhuman/tools/impl/system/` and consume this module's `NodeBootstrap`. ## Events -`ops::execute_tool` publishes (via `core::event_bus::publish_global`) around each bridge invocation, with `session_id = "javascript"`: +`ops::execute_tool` publishes around each bridge invocation, with +`session_id = "javascript"`: - `DomainEvent::ToolExecutionStarted` - `DomainEvent::ToolExecutionCompleted` (with `success`, `elapsed_ms`) -No event-bus subscribers (`bus.rs`) are defined. - ## Persistence -No domain `store.rs`. The only on-disk state is the **managed Node.js install cache**, resolved by `NodeBootstrap::cache_root()` (first hit wins): - -1. Explicit `config.node.cache_dir` (honoured verbatim). -2. `dirs::cache_dir()/openhuman/node-runtime` — the default, user-owned. -3. `{workspace}/node-runtime/` — last-resort fallback (warned; less secure). - -Reads `NodeConfig` (`config.node`): `enabled`, `prefer_system`, `version`, `cache_dir` (env overrides like `OPENHUMAN_NODE_ENABLED`, `OPENHUMAN_NODE_VERSION` in config loader). +**None here any more.** The managed install cache belongs to the `tinyruntime` +module, which decides where it lives from the settings each request carries. +This module reads `config.node` (`enabled`, `prefer_system`, `version`, +`cache_dir`) only to build those requests. ## Dependencies -- `crate::openhuman::config` (`Config`, `schema::NodeConfig`, `rpc`) — runtime config, version/cache settings, RPC config loading. -- `crate::openhuman::tools` (`Tool`, `ToolCallOptions`, `ToolScope`, `all_tools_with_runtime`) — the registry the bridge enumerates/executes. -- `crate::openhuman::security` (`SecurityPolicy`, workspace audit logger) — built per `build_runtime_tools` call to scope tool capability. -- `crate::openhuman::agent::host_runtime` (`NativeRuntime`, `RuntimeAdapter`) — runtime adapter injected into tool construction. -- `crate::openhuman::memory` / `memory_store` (`Memory`, `create_memory_with_local_ai`) — memory backend wired into memory-aware tools. -- `crate::openhuman::skills::types::ToolResult` — result envelope returned by `execute_tool`. -- `crate::openhuman::runtime::javascript` — the language-slot alias module the `rpc.rs` handlers call through (`list_tools` / `execute_tool`). -- `crate::core::event_bus` (`publish_global`, `DomainEvent`) — tool execution events. -- `crate::core::all` (`ControllerFuture`, `RegisteredController`) + `crate::core` (`ControllerSchema`, `FieldSchema`, `TypeSchema`) + `crate::rpc::RpcOutcome` — RPC controller plumbing. +- `crate::openhuman::modules::runtime` — the module client this delegates to. +- `crate::openhuman::config` — the settings each request carries. +- `crate::openhuman::tools`, `security`, `agent::host_runtime`, `memory` — the + registry the bridge enumerates and executes. +- `crate::core::event_bus`, `crate::core::all`, `crate::rpc` — events and RPC + plumbing. -External crates: `reqwest`, `sha2`, `hex`, `xz2`, `tar`, `zip`, `tokio`, `wait_timeout`, `dirs`, `anyhow`, `serde`/`serde_json`, `tracing`, `async-trait`. +External crates: `tinyruntime-bus`, `tokio`, `anyhow`, `serde`/`serde_json`, +`tracing`, `async-trait`. No HTTP client, no archive crates, no digest crate — +those went with the machinery. ## Used by -- `src/openhuman/runtime/javascript/mod.rs` — re-exports this entire surface under `javascript`-prefixed names (the public language slot). -- `src/openhuman/tools/impl/system/{node_exec,npm_exec,shell}.rs` — hold an `Arc`; `node_exec`/`npm_exec` call `resolve()`, `shell` uses non-blocking `try_cached()` for transparent `PATH` injection. -- `src/openhuman/tools/ops.rs` and `src/openhuman/agent/tools/delegate_to_personality.rs` — reference the bootstrap/runtime surface. -- `src/openhuman/runtime/python/bootstrap.rs` — a sibling runtime modeled on the same pattern. -- `src/core/all.rs` — registers the `javascript.*` controllers via the `javascript` aliases. +- `src/openhuman/runtime/javascript/mod.rs` — the public language slot. +- `src/openhuman/tools/impl/system/{node_exec,npm_exec,shell}.rs` — hold an + `Arc`; the exec tools call `resolve()`, `shell` uses the + non-blocking `try_cached()`. +- `src/openhuman/agent/harness_init/registry.rs` — the `node_runtime` init step + uses `probe_installed()` to decide whether provisioning is visible work. +- `src/core/all.rs` — registers the `javascript.*` controllers. ## Notes / gotchas -- **Naming asymmetry**: the module is `runtime_node` but its RPC namespace and public aliases are `javascript`. The `javascript` module is a deliberate language-slot indirection so a future backend (or `python`/`ruby`) can swap in without churning callers. -- **`build_runtime_tools` is not cheap**: each `list_tools`/`execute_tool` call rebuilds the full tool registry (security policy, audit logger, memory backend) from `Config`. There is no caching at the bridge layer — the memoisation in `bootstrap.rs` is only for Node toolchain resolution, not for tool construction. -- **Integrity is load-bearing, no opt-out**: downloads must match the official `SHASUMS256.txt` digest or the archive is deleted and the call fails. `probe_managed_install` canonicalises and requires the install to live under the resolved cache root to defeat a workspace-vendored fake `node-v*/` tree (PR #723 finding). -- **System-node reuse requires npm**: a compatible `node` with a missing/broken `npm` is rejected so the managed path can supply a complete toolchain (distros that split `nodejs`/`npm`). -- **Version match is major-only** (`parse_node_version`); point releases are accepted. Set `node.prefer_system = false` for strict pinning, or `node.enabled = false` to disable the runtime entirely (then `resolve()` bails). -- **Self-healing cache**: a managed install missing `npm` (e.g. download interrupted after `node` extracted) is treated as unusable and reinstalled rather than reused forever. +- **Naming asymmetry**: the directory is `node` but its RPC namespace and public + aliases are `javascript`. That indirection is what let the backend underneath + be replaced by a bus module without churning a single caller. +- **`build_runtime_tools` is not cheap**: each bridge call rebuilds the full tool + registry from `Config`. There is no caching at the bridge layer — the + memoisation here is only for toolchain resolution. +- **The local cache is not redundant with the module's.** The module memoises + too, but only this one can answer without awaiting, which is the entire reason + the shell can inject `PATH` without blocking. +- **Version policy lives in the provider now.** Major-only matching, and the + `prefer_system = false` escape hatch for strict pinning, are decisions of + `tinyruntime-nodejs`; this module carries the setting rather than the rule. +- **A toolchain without `npm` still resolves.** `npm_bin` is derived when the + provider does not report it: refusing would take `node_exec` down along with + `npm_exec`, for an install that runs `node` perfectly well. diff --git a/src/openhuman/runtime/python/README.md b/src/openhuman/runtime/python/README.md index e12637fadb..4f86ca0347 100644 --- a/src/openhuman/runtime/python/README.md +++ b/src/openhuman/runtime/python/README.md @@ -1,78 +1,78 @@ -# runtime_python +# runtime/python -Managed Python runtime for Python-backed integrations. This domain owns interpreter discovery and process-launch primitives so callers don't need to care whether Python came from the host or a managed standalone CPython distribution. The immediate use case is launching stdio MCP servers implemented in Python. The shipped intent is a managed CPython distribution downloaded from `astral-sh/python-build-standalone`; a system-interpreter probe is a compatibility / developer override path. +The **Python interpreter client**, plus the process-launch helper for the +long-lived Python children this core owns. -## Responsibilities +## What moved out -- Resolve a Python ≥ `minimum_version` (default `3.12.0`) interpreter, memoizing the first success. -- Optionally probe host `PATH` for a compatible interpreter (`prefer_system`). -- Download, SHA-256-verify, extract, and atomically install a managed standalone CPython distribution when no system Python is used/available. -- Spawn line-oriented (unbuffered) Python child processes for stdio protocols such as MCP. -- Read its behavior from `[runtime_python]` config (`RuntimePythonConfig`). +Interpreter discovery (candidate ordering, `--version` probing, minimum-version +matching) and the managed standalone-CPython install pipeline (release index +selection, download, digest verification, extraction, atomic install, +cross-process install locking) are all in the `tinyruntime` module now, reached +through [`modules::runtime`](../../modules/runtime.rs). + +## What stayed, and why + +`process.rs` launches stdio Python children — the runtime Python server, and the +stdio MCP servers. That is deliberately **not** the module's pooled execution: +those children outlive a single job, speak their own protocols, and are owned by +the subsystem that started them. The module resolves the interpreter; this core +decides what to run with it. ## Key files | File | Role | | --- | --- | -| `src/openhuman/runtime/python/mod.rs` | Export-focused module root; module docstring + `pub use` re-exports of the public surface. | -| `src/openhuman/runtime/python/bootstrap.rs` | Orchestrator. `PythonBootstrap` ties resolve → (system probe \| managed install) → memoized `ResolvedPython`; exposes `spawn_stdio`. Holds the per-install file lock, cache-root selection, and managed-install probing. | -| `src/openhuman/runtime/python/resolver.rs` | System Python discovery. Walks candidate commands / `PATH`, probes `--version` (5s timeout), parses semver, enforces the minimum-version floor. | -| `src/openhuman/runtime/python/downloader.rs` | Managed distribution fetch. Queries the GitHub releases API, selects a host-compatible `install_only` asset ≥ minimum, downloads and verifies the published SHA-256 digest. | -| `src/openhuman/runtime/python/extractor.rs` | `tar.gz` extraction (gzip via `flate2`, preserves perms) + `atomic_install` (rename-into-place with backup/rollback). | -| `src/openhuman/runtime/python/process.rs` | `PythonLaunchSpec` + `spawn_stdio_process`: builds a `tokio::process::Command` with piped stdio, `-u`, `kill_on_drop`. | -| `src/openhuman/runtime/python/bootstrap_tests.rs` | Tests for the bootstrap orchestrator (`#[path]`-included). | -| `src/openhuman/runtime/python/resolver_tests.rs` | Tests for version parsing / system detection. | -| `src/openhuman/runtime/python/downloader_tests.rs` | Tests for release-metadata parse and asset selection. | +| `mod.rs` | Export-focused: submodule decls and `pub use` re-exports. | +| `bootstrap.rs` | The interpreter client. `PythonBootstrap` (`resolve`, `probe_installed`, `try_cached`, `spawn_stdio`), `ResolvedPython`, `PythonSource`. | +| `process.rs` | `PythonLaunchSpec` and `spawn_stdio_process`: unbuffered stdio (`-u`), piped fds, `kill_on_drop`, and the Windows no-console flag. | ## Public surface -Re-exported from `mod.rs`: - -- `bootstrap`: `PythonBootstrap`, `PythonSource` (`System` / `Managed`), `ResolvedPython` (`python_bin`, `version`, `source`). -- `downloader`: `fetch_release_metadata`, `select_distribution`, `PythonDistribution`. -- `extractor`: `atomic_install`, `extract_distribution`. -- `process`: `PythonLaunchSpec`. -- `resolver`: `detect_system_python`, `parse_python_version`, `PythonVersion`, `SystemPython`. - -Primary entry points: `PythonBootstrap::new(config)`, `.resolve() -> Result`, `.try_cached()`, `.spawn_stdio(&PythonLaunchSpec) -> Result`. - -## RPC / controllers - -None. This domain exposes no JSON-RPC controllers, schemas, or `handle_*` functions — it is a library used by other domains, not directly addressable over RPC. - -## Agent tools - -None. No `tools.rs`; the module owns no agent tools. - -## Events - -None. No `bus.rs`; the module neither publishes nor subscribes to `DomainEvent`s. +`PythonBootstrap::new(Arc)`, `.resolve() -> Result`, +`.probe_installed()`, `.try_cached()`, +`.spawn_stdio(&PythonLaunchSpec) -> Result`, plus +`ResolvedPython` (`python_bin`, `bin_dir`, `version`, `source`) and +`PythonSource`. ## Persistence -No structured domain store. Side effects on disk: - -- Managed CPython installs land under the cache root: `config.cache_dir` if set, else `/openhuman/runtime-python`, else `.openhuman/runtime-python`. -- Per-install exclusive file lock at `.lock` (`fs2`) serializes concurrent installs. -- Staging dirs (`.stage--`) and the downloaded archive are removed after a successful atomic install. Existing installs are moved aside to `.old-` and restored on rename failure. +**None here.** The managed install cache belongs to the `tinyruntime` module. +This module reads `config.runtime_python` (`enabled`, `prefer_system`, +`minimum_version`, `maximum_version`, `cache_dir`, `managed_release_tag`, +`preferred_command`) only to build the requests it sends. ## Dependencies -- `crate::openhuman::config::schema::RuntimePythonConfig` — the only intra-crate dependency; drives `enabled`, `minimum_version`, `cache_dir`, `managed_release_tag`, `prefer_system`, `preferred_command`. +- `crate::openhuman::modules::runtime` — the module client this delegates to. +- `crate::openhuman::config` — the settings each request carries. +- `crate::openhuman::inference::local::process_util` — the Windows no-console + hook, shared with the other child-spawning paths. -External crates: `reqwest` (HTTP), `serde` (release metadata), `sha2`/`hex` (digest verify), `flate2`/`tar` (extraction), `tokio` (async fs/process + `Mutex`), `fs2` (file lock), `walkdir` (interpreter discovery), `wait-timeout` (version probe timeout), `uuid`, `dirs`, `anyhow`, `tracing`. +External crates: `tinyruntime-bus`, `tokio`, `anyhow`, `tracing`. No HTTP +client, no archive crates, no `walkdir`, no `fs2` — those went with the pipeline. ## Used by -Referenced from `src/openhuman/mod.rs` (module declaration) and surfaced in the capability catalog (`src/openhuman/platform/about_app/catalog.rs`). Config wiring lives in `src/openhuman/config/schema/{runtime_python.rs,types.rs,load.rs,mod.rs}`. No other domain currently constructs `PythonBootstrap` directly in `src/` outside this wiring — the intended consumer is Python-backed integrations such as stdio MCP servers. +- `src/openhuman/runtime/python_server/` — resolves an interpreter, then spawns + and supervises the long-lived model server with `spawn_stdio`. +- `src/openhuman/tools/impl/system/{python_exec,shell}.rs` — hold an + `Arc`; `python_exec` calls `resolve()`, `shell` uses the + non-blocking `try_cached()` for `PATH` injection. +- `src/openhuman/skills/runtime/ops.rs` — resolves an interpreter for + Python-backed skills. +- `src/openhuman/agent/harness_init/registry.rs` — the Python init step uses + `probe_installed()` to decide whether provisioning is visible work. ## Notes / gotchas -- `resolve()` is memoized: the first successful `ResolvedPython` is cached behind a `tokio::Mutex` and returned to all later callers; `try_cached()` peeks without probing. -- When `config.enabled == false`, `resolve()` bails — callers must skip Python-backed features rather than fall back. -- Managed install is only attempted when `prefer_system` is off or the system probe finds nothing compatible; the system path returns `PythonSource::System`, managed returns `PythonSource::Managed`. -- Host-asset selection prefers `install_only_stripped` assets when available; only the platform triples enumerated in `host_asset_suffix` are supported (macOS/Linux/Windows × x86_64/aarch64) — other hosts error. -- Download verification: if release metadata lacks a `digest`, SHA-256 verification is **skipped** with a warning rather than failing. -- The version probe runs ` --version` with a 5s timeout, `CREATE_NO_WINDOW` on Windows, and reads version output from stdout or (fallback) stderr. -- `spawn_stdio_process` defaults to `-u` (unbuffered) and `kill_on_drop(true)` so dropped children don't leak; stdin/stdout/stderr are all piped. -- `bootstrap.rs` defines a private `install_managed()` wrapper that is currently unused by the public path (`resolve()` calls `install_managed_from_api` directly). +- **A request names a floor, not a version.** `runtime_python.minimum_version` + is a lower bound because the standalone channel publishes a moving set of + builds; the exclusive `maximum_version` is how a host stays off a newer + series. Both are interpreted by `tinyruntime-python`, not here. +- **The local cache is not redundant with the module's.** Only this one can + answer without awaiting, which is what lets the shell inject `PATH` without + blocking on a bus round trip. +- **`spawn_stdio` is not pooled execution.** It exists for children that outlive + a job. Inline Python code goes through `runtime::pool::python` instead, which + routes to the module's warm workers. From b178896bb8fca8b9e6a69dad6e35ed6d0f49fdbb Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:21:56 +0300 Subject: [PATCH 31/55] chore: reorder imports and reformat long assertions Reordered `anyhow` imports to place `anyhow` before `Result` for consistency across bootstrap and stub files, and reformatted several test assertions to comply with the project's line-length conventions. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/bootstrap.rs | 2 +- src/openhuman/runtime/node/bootstrap_tests.rs | 19 +++++++++++++++---- src/openhuman/runtime/node/mod.rs | 2 +- src/openhuman/runtime/node/stub.rs | 2 +- src/openhuman/runtime/pool/pool_tests.rs | 12 +++++++++--- src/openhuman/runtime/pool/types_tests.rs | 5 ++++- src/openhuman/runtime/python/bootstrap.rs | 2 +- .../runtime/python/bootstrap_tests.rs | 4 +++- 8 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/openhuman/runtime/node/bootstrap.rs b/src/openhuman/runtime/node/bootstrap.rs index 3a2acb4d65..c90a3f2743 100644 --- a/src/openhuman/runtime/node/bootstrap.rs +++ b/src/openhuman/runtime/node/bootstrap.rs @@ -26,7 +26,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; -use anyhow::{Result, anyhow}; +use anyhow::{anyhow, Result}; use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; use crate::openhuman::config::Config; diff --git a/src/openhuman/runtime/node/bootstrap_tests.rs b/src/openhuman/runtime/node/bootstrap_tests.rs index 55850f5bd1..70e4c855f9 100644 --- a/src/openhuman/runtime/node/bootstrap_tests.rs +++ b/src/openhuman/runtime/node/bootstrap_tests.rs @@ -29,9 +29,18 @@ fn a_resolution_becomes_the_paths_callers_name() { ])) .expect("a toolchain reporting node adapts"); - assert_eq!(adapted.node_bin, std::path::Path::new("/cache/node-v22.11.0/bin/node")); - assert_eq!(adapted.npm_bin, std::path::Path::new("/cache/node-v22.11.0/bin/npm")); - assert_eq!(adapted.bin_dir, std::path::Path::new("/cache/node-v22.11.0/bin")); + assert_eq!( + adapted.node_bin, + std::path::Path::new("/cache/node-v22.11.0/bin/node") + ); + assert_eq!( + adapted.npm_bin, + std::path::Path::new("/cache/node-v22.11.0/bin/npm") + ); + assert_eq!( + adapted.bin_dir, + std::path::Path::new("/cache/node-v22.11.0/bin") + ); assert_eq!(adapted.version, "22.11.0"); assert_eq!(adapted.source, NodeSource::Managed); } @@ -44,7 +53,9 @@ fn a_toolchain_without_npm_still_resolves() { .expect("a toolchain without npm is still usable"); assert_eq!(adapted.node_bin, std::path::Path::new("/usr/bin/node")); assert!( - adapted.npm_bin.ends_with(if cfg!(windows) { "npm.cmd" } else { "npm" }), + adapted + .npm_bin + .ends_with(if cfg!(windows) { "npm.cmd" } else { "npm" }), "npm was not derived: {}", adapted.npm_bin.display() ); diff --git a/src/openhuman/runtime/node/mod.rs b/src/openhuman/runtime/node/mod.rs index f0575c8892..ddc4f4534d 100644 --- a/src/openhuman/runtime/node/mod.rs +++ b/src/openhuman/runtime/node/mod.rs @@ -38,7 +38,7 @@ pub mod types; #[cfg(not(feature = "runtime-node"))] mod stub; #[cfg(not(feature = "runtime-node"))] -pub use stub::{NodeBootstrap, NodeSource, RUNTIME_NODE_DISABLED_MESSAGE, ResolvedNode}; +pub use stub::{NodeBootstrap, NodeSource, ResolvedNode, RUNTIME_NODE_DISABLED_MESSAGE}; #[cfg(feature = "runtime-node")] pub use bootstrap::{NodeBootstrap, NodeSource, ResolvedNode}; diff --git a/src/openhuman/runtime/node/stub.rs b/src/openhuman/runtime/node/stub.rs index 5e6ff405cb..1656ed7131 100644 --- a/src/openhuman/runtime/node/stub.rs +++ b/src/openhuman/runtime/node/stub.rs @@ -25,7 +25,7 @@ use std::path::PathBuf; use std::sync::Arc; -use anyhow::{Result, anyhow}; +use anyhow::{anyhow, Result}; use crate::openhuman::config::Config; diff --git a/src/openhuman/runtime/pool/pool_tests.rs b/src/openhuman/runtime/pool/pool_tests.rs index 75ef2f430b..52bb00b7f0 100644 --- a/src/openhuman/runtime/pool/pool_tests.rs +++ b/src/openhuman/runtime/pool/pool_tests.rs @@ -5,7 +5,7 @@ //! whether a language pools by default, and how a module failure is classified, //! because that classification is what keeps a job from running twice. -use super::{PoolRunError, classify, node, python}; +use super::{classify, node, python, PoolRunError}; use crate::openhuman::config::Config; use crate::openhuman::modules::runtime::RuntimeCallError; @@ -75,9 +75,15 @@ fn anything_else_is_pre_dispatch_so_the_caller_may_fall_back() { #[test] fn the_three_classifications_render_distinguishably() { // They drive opposite caller behaviour, so their messages must not blur. - assert_eq!(PoolRunError::Saturated.to_string(), "runtime pool at capacity"); + assert_eq!( + PoolRunError::Saturated.to_string(), + "runtime pool at capacity" + ); let pre = PoolRunError::PreDispatch(anyhow::anyhow!("spawn failed")).to_string(); assert!(pre.starts_with("pre-dispatch pool failure:"), "got {pre}"); let post = PoolRunError::PostDispatch(anyhow::anyhow!("read wedged")).to_string(); - assert!(post.starts_with("post-dispatch pool failure:"), "got {post}"); + assert!( + post.starts_with("post-dispatch pool failure:"), + "got {post}" + ); } diff --git a/src/openhuman/runtime/pool/types_tests.rs b/src/openhuman/runtime/pool/types_tests.rs index da5c4b7991..5286babd62 100644 --- a/src/openhuman/runtime/pool/types_tests.rs +++ b/src/openhuman/runtime/pool/types_tests.rs @@ -28,7 +28,10 @@ fn success_requires_a_clean_exit_and_no_timeout() { timed_out: true, ..outcome(Some(0)) }; - assert!(!timed_out.success(), "a job aborted at its deadline did not succeed"); + assert!( + !timed_out.success(), + "a job aborted at its deadline did not succeed" + ); } #[test] diff --git a/src/openhuman/runtime/python/bootstrap.rs b/src/openhuman/runtime/python/bootstrap.rs index 8dcc94d50c..04b1bc0f24 100644 --- a/src/openhuman/runtime/python/bootstrap.rs +++ b/src/openhuman/runtime/python/bootstrap.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; -use anyhow::{Result, anyhow}; +use anyhow::{anyhow, Result}; use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; use crate::openhuman::config::Config; diff --git a/src/openhuman/runtime/python/bootstrap_tests.rs b/src/openhuman/runtime/python/bootstrap_tests.rs index c72b41a7d6..78aac8a1ea 100644 --- a/src/openhuman/runtime/python/bootstrap_tests.rs +++ b/src/openhuman/runtime/python/bootstrap_tests.rs @@ -57,7 +57,9 @@ fn a_system_interpreter_is_reported_as_one() { let mut resolved = resolution(&[("python", "/usr/bin/python3")]); resolved.source = RuntimeSource::System; assert_eq!( - ResolvedPython::from_module(&resolved).expect("adapts").source, + ResolvedPython::from_module(&resolved) + .expect("adapts") + .source, PythonSource::System ); } From e63bf4a2c5ab9fe1a35a19012781477aac237547 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:28:14 +0300 Subject: [PATCH 32/55] docs(library-minimal-recipe, specs): update xz2 removal and runtime bootstrap deps The Node/Python runtime bootstrap dependencies section in the minimal recipe is updated to reflect that xz2 and its liblzma build have been removed from the manifest entirely, as downloading and unpacking language toolchains moved into the tinyruntime module. The corresponding entry in the core kernel domain reorg spec is also updated to mark the lzma-sys removal as complete and better than originally planned. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/library-minimal-recipe.md | 10 ++++++---- docs/specs/2026-08-02-core-kernel-domain-reorg.md | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/library-minimal-recipe.md b/docs/library-minimal-recipe.md index df1db1094c..97bc830e2b 100644 --- a/docs/library-minimal-recipe.md +++ b/docs/library-minimal-recipe.md @@ -228,10 +228,12 @@ prioritization. **and** `native-tls` — two full TLS stacks linked simultaneously. A headless host on a known target could pick one, shedding the other. -5. **Node/Python runtime bootstrap deps** (`tar`, `xz2`+liblzma, `zip`, `flate2`). - Only needed if `skills`/`flows` actually execute node/python workloads; kept - here because `skills` is on. If a deployment runs only pure-LLM skills, these - archive/decompression deps become sheddable. +5. ~~**Node/Python runtime bootstrap deps**~~ — **no longer applicable.** + Downloading and unpacking language toolchains moved into the `tinyruntime` + module, so `xz2` and its liblzma build left this manifest entirely. `tar`, + `zip`, and `flate2` remain, but for the Piper voice installer and the document + tools rather than for any runtime bootstrap; they are sheddable with those + features, not with `skills`. ## See also diff --git a/docs/specs/2026-08-02-core-kernel-domain-reorg.md b/docs/specs/2026-08-02-core-kernel-domain-reorg.md index 5dab34d91e..ac976b587e 100644 --- a/docs/specs/2026-08-02-core-kernel-domain-reorg.md +++ b/docs/specs/2026-08-02-core-kernel-domain-reorg.md @@ -265,7 +265,7 @@ the reorg: | Native crate | Owner | Gate | Status | | --- | --- | --- | --- | | `libgit2-sys` (via `git2`) | `memory_store/content/wiki_git` **and** `tinycortex/git-diff` | `memory-git` | **cross-repo** — see below | -| `lzma-sys` (via `xz2`) | `runtime_node/extractor.rs` only | `runtime-node` | ready; do with the `runtime/` move | +| `lzma-sys` (via `xz2`) | *(none — removed)* | — | ✅ **done**, and better than planned: extraction moved into the `tinyruntime` module, so `xz2` left the manifest for **every** configuration rather than only for builds that opted out of `runtime-node` | | `libz-sys` | shared (`flate2`/`zip`/`git2`) | — | partly falls out of the above | | `aws-lc-sys` | TLS stack | — | needs a rustls-provider decision, own slice | | `libsqlite3-sys`, `ring` | kernel | — | **target keeps these** | From f70d41e41d5221961696a6df281334eb00d7a97d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:28:30 +0300 Subject: [PATCH 33/55] docs(library-benchmarking): document worker relocation to tinyruntime module Adds a note explaining that pooled workers now live in the tinyruntime module rather than in-process, clarifying that the resident cost measured by the benchmark has moved with them while configuration and behaviour remain unchanged. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/library-benchmarking.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/library-benchmarking.md b/docs/library-benchmarking.md index 5143174556..777506d2b7 100644 --- a/docs/library-benchmarking.md +++ b/docs/library-benchmarking.md @@ -367,6 +367,12 @@ The pool is configured in `[runtime_pool]` (master switch + per-language `max_queue_depth`); `enabled = false` reverts every caller to the legacy per-call spawn. +The workers themselves now live in the `tinyruntime` module rather than in this +process, so the resident cost this measures has moved with them: a pooled worker +is still one interpreter child of the host, but it is spawned and supervised +across the bus. The configuration keys, the backpressure behaviour, and the +`enabled = false` escape hatch are unchanged. + Watch-items from the sweep: thread count grows ~0.35/agent (needs attribution + cap before real 1000-agent runs), and p95 latency at N=500 on 2 workers shows CPU saturation is the load constraint, not memory. From 7e93ca7f8a08774be9dd6c6223eed671a545e801 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:28:53 +0300 Subject: [PATCH 34/55] docs(AGENTS.md): update runtime-node and skills dep notes to reflect tinyruntime module Update the `runtime-node` gate description to clarify that the client now asks the `tinyruntime` module for a Node.js toolchain, and note that the `xz2` native build has been removed from all configurations rather than only slim ones. Also fix the `runtime::node`/`runtime::python` path references in the skills dep note to use the correct module path syntax. Auto-committed-on: dragonfly Co-authored-by: Medulla --- AGENTS.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a3a61df331..179e447572 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -349,7 +349,7 @@ two paths' equivalence — keep that as call sites migrate. A move never changes the wire surface — RPC namespaces are string literals in `ControllerSchema`, not derived from module paths — so **do not rename namespace strings to match new paths**. -**Skills runtime**: the QuickJS per-skill VM engine is gone. `src/openhuman/skills/` holds skill metadata/tool descriptors; execution of installed `SKILL.md` workflows lives in `src/openhuman/skills/runtime/` (starts/cancels runs, hosts the `skill_executor` agent, reuses `runtime_node`/`runtime_python`). +**Skills runtime**: the QuickJS per-skill VM engine is gone. `src/openhuman/skills/` holds skill metadata/tool descriptors; execution of installed `SKILL.md` workflows lives in `src/openhuman/skills/runtime/` (starts/cancels runs, hosts the `skill_executor` agent, reuses `runtime::node`/`runtime::python`, which are clients for the `tinyruntime` module). **Rules:** @@ -515,7 +515,7 @@ Two columns because there are two sets (see above): **Contrib** is `[features] d | `channels` | ON | ON | `openhuman::channels` (external-messaging providers — Telegram/Discord/Slack/Signal/WhatsApp/iMessage/IRC/… — plus the channel runtime, controllers, host, proactive messaging + inbound dispatch) and the `channels::webview_accounts` / `webview_apis` / `webview_notifications` / `channels::whatsapp_data` webview-bridge domains (incl. the 3 `whatsapp_data_*` agent tools). **Carve-outs `channels::{traits, cli}` stay ungated.** | **28** via `tinychannels/{email,lark}` — the crate itself stays (load-bearing), its two heavy providers do not | | `memory-git` | OFF | ON | `openhuman::memory::diff` (git-backed snapshots/checkpoints/read markers, the `memory_diff` RPC namespace + agent tool) and the git wiki mirror in `memory::store::content::wiki_git`. **Type carve-out**: `memory::diff::types` compiles in BOTH builds — the always-on subconscious memory profile renders `CrossSourceDiff`/`ChangeKind` into prompts, and tinycortex makes the matching split (its `memory::diff::{types,source}` are ungated, only the `Ledger`/`DiffEngine` half sits behind `git-diff`). Off ⇒ `memory_diff` is unknown-method, the tool is absent, the embedded driver drops `Capability::Diff` **and** `as_diff()` returns `None` in lockstep (`audit_provider` fails on either half alone), and summary nodes are still written to disk but not mirrored into git. **This crate declares no `git2`** — tinycortex owns every libgit2 call in the stack (the diff ledger, the wiki mirror, the persona git-history reader), and the gate reaches the cohort by forwarding `tinycortex/git-diff` + `tinycortex/wiki-git`; `tinymemory-core/memory-git` forwards the same pair. Do not re-add a direct `git2` dependency to this crate or to `tinymemory-core`: it would buy no crates and invite a second major pin, which `links = "git2"` makes a hard cargo error. Test code that must read a ledger back goes through the `tinycortex::git2` re-export (`tests/memory_artifacts_e2e.rs`). | **3**: `git2`, `libgit2-sys`, `libz-sys` — two of the five native C builds in the kernel profile, the largest native shed in the program | | `contacts` | OFF | ON | `memory::people::address_book`'s macOS CNContactStore reader — the address-book seeding path for the people domain. Leaf gate over a **pre-existing** off-state: the module already shipped a non-macOS `imp` stub returning an empty contact list, so the gate only widens that stub's cfg. `read`/`read_with`/`AddressBookError`/`SystemContactsSource` and the whole `people` RPC surface stay compiled in every build; off ⇒ a refresh seeds nothing instead of failing. | **6** on macOS (`objc2`, `objc2-foundation`, `objc2-contacts`, `block2` + 2 transitive). **No-op on Linux/Windows** — never in those graphs, so the kernel-floor ratchet does not move. Verify cross-target: `cargo tree --target aarch64-apple-darwin -e normal -i objc2-contacts --no-default-features` (294 → 288 packages). | -| `runtime-node` | OFF | ON | `runtime::node` (download / verify / extract / install a pinned Node.js toolchain), the `runtime::javascript` language slot, `runtime::pool::node`, the `node_exec` / `npm_exec` agent tools, and the `node_runtime` harness-init step. **Facade + stub** — `ShellTool` holds `Option>` and `shell.rs` is kernel, so the module cannot simply vanish; `runtime/node/stub.rs` carries the `NodeBootstrap` type surface while registration sites are leaf-gated. **The generic native-tool dispatcher (`runtime::node::ops` / `runtime::node::types`) is NOT gated** — it backs both the gated `javascript.*` controllers and the ungated `flows` `oh:` `NativeToolBackend`, so native flow tools (`memory_search`, file, shell, …) keep working when the managed Node runtime is off. Off ⇒ `try_cached`/`probe_installed` return `None` and the shell never prepends a managed bin dir, identical to today's `node.enabled = false` path. | **`xz2` + its static liblzma C build.** First gate to remove a NATIVE toolchain build: `lzma-sys` leaves the list, 6 → 5. `tar`/`zip` are NOT shed — shared with `inference` (install_piper), `runtime::python`, and the document tools. | +| `runtime-node` | OFF | ON | `runtime::node` (the client that asks the `tinyruntime` module for a Node.js toolchain), the `runtime::javascript` language slot, `runtime::pool::node`, the `node_exec` / `npm_exec` agent tools, and the `node_runtime` harness-init step. **Facade + stub** — `ShellTool` holds `Option>` and `shell.rs` is kernel, so the module cannot simply vanish; `runtime/node/stub.rs` carries the `NodeBootstrap` type surface while registration sites are leaf-gated. **The generic native-tool dispatcher (`runtime::node::ops` / `runtime::node::types`) is NOT gated** — it backs both the gated `javascript.*` controllers and the ungated `flows` `oh:` `NativeToolBackend`, so native flow tools (`memory_search`, file, shell, …) keep working when the managed Node runtime is off. Off ⇒ `try_cached`/`probe_installed` return `None` and the shell never prepends a managed bin dir, identical to today's `node.enabled = false` path. | **Nothing any more.** This gate used to shed `xz2` and its static liblzma C build; download and extraction moved into the `tinyruntime` module, so that native build left the manifest for **every** configuration rather than only for slim ones. The gate still buys the absence of the tools and controllers. | **Facade pattern (pathfinder for the other gates).** `pub mod voice;` is **always compiled** as a facade: the real submodules are `#[cfg(feature = "voice")]`, and a `#[cfg(not(feature = "voice"))] mod stub;` (`src/openhuman/voice/stub.rs`) re-exposes the same public surface that always-on / other-gated callers use (`server`, `dictation_listener`, `streaming`, `reply_speech`, `cloud_transcribe`, `cli`, `create_stt_provider`, `effective_stt_provider`, `publish_ptt_transcript_committed`) with no-op / `None` / disabled-error bodies. Callers therefore do **not** need per-call `#[cfg]`. When voice is off: the voice/audio controllers are unregistered (unknown-method over `/rpc`, absent from `/schema`), the `audio_generate_podcast` agent tools are absent, and `openhuman voice` returns a "voice disabled" error. Stub signatures must match the real ones exactly — the disabled build (`--no-default-features`) is the **only** thing that catches drift, so run it before pushing any change to the voice surface. @@ -560,7 +560,7 @@ Two places the carve-out doesn't reach, and why they are `#[cfg]` at the call si - `agent/registry/agents/loader.rs` — the `skill_setup` / `skill_executor` `BuiltinAgent` entries. `include_str!` embeds the agent TOML from disk regardless of module gating, so the entry itself must disappear. - `agent/task_dispatcher/executor.rs` — the workflow-resolution branch. `registry::get_workflow` returns `Option`, which flattens in `AgentDefinition` and is destructured at the call site; stubbing it would mean re-declaring that struct (exactly what the carve-out avoids). With the domain compiled out no handle can resolve to a skill, so falling through to the builtin-agent branch is correct, not degraded. -**Dep note:** `skills = []` — the empty list is **intentional, do not "fix" it**. Unlike `voice` (`hound`/`lettre`), these domains have no exclusive dependencies: every crate they touch is shared with always-on domains, and `runtime_node` / `runtime_python` are used by Agent / Flows / Memory too. This gate's value is tool-surface + prompt-bloat + startup cost, **not** binary size. +**Dep note:** `skills = []` — the empty list is **intentional, do not "fix" it**. Unlike `voice` (`hound`/`lettre`), these domains have no exclusive dependencies: every crate they touch is shared with always-on domains, and `runtime::node` / `runtime::python` are used by Agent / Flows / Memory too. This gate's value is tool-surface + prompt-bloat + startup cost, **not** binary size. When skills are off: the `skills` / `skill_runtime` / `skill_registry` controllers are unregistered (unknown-method over `/rpc`, absent from `/schema`), the 16 skill agent tools (incl. `run_workflow` / `await_workflow`) are **absent** from the tool list rather than degraded to an error, the `skill_setup` / `skill_executor` builtin agents are gone, and the boot-time remote catalog refresh is skipped. Composes with the runtime `DomainSet::skills` flag (#4796) — that axis needed no change here; #4798 is compile-time only. From d00102871a42f817258171d822708313872735d9 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:34:40 +0300 Subject: [PATCH 35/55] docs(config): clarify NodeConfig field semantics Expand doc comments on `NodeConfig` fields to document runtime behaviour: the `enabled` flag is checked before any bus call, `version` is matched on the major component by the `tinyruntime` module, `cache_dir` is owned by that module, and `prefer_system` controls whether a system node is reused or a managed toolchain is installed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/config/schema/node.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/openhuman/config/schema/node.rs b/src/openhuman/config/schema/node.rs index e6f64223e4..fe945600f3 100644 --- a/src/openhuman/config/schema/node.rs +++ b/src/openhuman/config/schema/node.rs @@ -10,21 +10,27 @@ use serde::{Deserialize, Serialize}; #[serde(default)] pub struct NodeConfig { /// Master switch. When `false`, the Node runtime is not resolved and - /// `node_exec` / `npm_exec` tools are not registered. + /// `node_exec` / `npm_exec` tools are not registered. Checked before any + /// bus call, so a disabled runtime costs nothing. #[serde(default = "default_enabled")] pub enabled: bool, - /// Target Node.js release line (used to build download URLs and bin cache - /// directory name, e.g. `v22.11.0`). Pin to a known LTS for reproducibility. + /// Target Node.js release line, e.g. `v22.11.0`. Pin to a known LTS for + /// reproducibility. + /// + /// Carried to the `tinyruntime` module on every request; the Node provider + /// there matches on the **major** component, so a host `v22.8.0` satisfies + /// `v22.11.0`. Set `prefer_system = false` for an exact toolchain. #[serde(default = "default_version")] pub version: String, /// Absolute path to a directory where managed Node distributions are - /// extracted. Empty string means "use the default OpenHuman cache dir" - /// (resolved by the runtime bootstrap). + /// installed. Empty string means "use the platform cache directory" + /// (resolved by the `tinyruntime` module, which owns the install). #[serde(default)] pub cache_dir: String, /// When `true` and a system `node` binary is found on `PATH` whose major - /// version matches `version`, reuse it instead of downloading. Disable for - /// reproducible CI / airgapped deployments. + /// version matches `version`, reuse it instead of installing a managed + /// toolchain. Disable for reproducible CI / airgapped deployments — that is + /// also how a caller pins an exact version rather than a major line. #[serde(default = "default_prefer_system")] pub prefer_system: bool, } From 391419c4033480ef1340b0f85d5c55262eda5d93 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:35:41 +0300 Subject: [PATCH 36/55] docs(library-benchmarking): clarify that the pool relocation does not affect the benchmark The documentation for the benchmarking scenario is updated to explain that moving the pool into the `tinyruntime` module does not change what the test measures, because a TinyBus module is loaded into the host process and any worker it spawns remains a child of the host. The previous wording incorrectly suggested the resident cost had moved with the workers. Auto-committed-on: dragonfly Co-authored-by: Medulla --- docs/library-benchmarking.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/library-benchmarking.md b/docs/library-benchmarking.md index 777506d2b7..445ff3e68b 100644 --- a/docs/library-benchmarking.md +++ b/docs/library-benchmarking.md @@ -367,10 +367,11 @@ The pool is configured in `[runtime_pool]` (master switch + per-language `max_queue_depth`); `enabled = false` reverts every caller to the legacy per-call spawn. -The workers themselves now live in the `tinyruntime` module rather than in this -process, so the resident cost this measures has moved with them: a pooled worker -is still one interpreter child of the host, but it is spawned and supervised -across the bus. The configuration keys, the backpressure behaviour, and the +The pool itself now lives in the `tinyruntime` module. That does **not** change +what this scenario measures: a TinyBus module is a `cdylib` loaded into this +process, so a worker it spawns is still a child of the host and still shows up in +the process-tree sample the gate asserts on. What changed is which code spawns +it. The configuration keys, the backpressure behaviour, and the `enabled = false` escape hatch are unchanged. Watch-items from the sweep: thread count grows ~0.35/agent (needs From c621a27b387b593e09ab39828c6d17ae5d44b069 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 21:36:22 +0300 Subject: [PATCH 37/55] docs(runtime): add README for pool module Added a README file to document the pool module, providing an overview of its purpose and usage within the runtime. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/pool/README.md | 53 ++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/openhuman/runtime/pool/README.md diff --git a/src/openhuman/runtime/pool/README.md b/src/openhuman/runtime/pool/README.md new file mode 100644 index 0000000000..8a4ebf175c --- /dev/null +++ b/src/openhuman/runtime/pool/README.md @@ -0,0 +1,53 @@ +# runtime/pool + +The client for pooled inline execution. The pool itself — warm interpreter +children, the job protocol, backpressure, idle reaping, recycle-after-N — lives +in the `tinyruntime` module, where one implementation serves every language. + +What is here is two decisions this core still owns. + +## 1. Whether a language pools at all + +| Language | Default | Why | +| --- | --- | --- | +| `node` | **on** | Each job runs in its own `worker_thread`: a fresh module graph and fresh globals per job, so reuse is safe. | +| `python` | **off** | Jobs share one interpreter. CPython has no worker-thread equivalent and no safe way to kill a running thread, so reuse leaks `sys.modules`, `os.environ`, logging handlers, and threads across unrelated runs. Opt in with `[runtime_pool.python] enabled = true`. | + +`[runtime_pool] enabled = false` is the master switch and reverts every caller to +its legacy per-call spawn, with no behavioural change. Pooling is an optimisation +seam, not a dependency. + +## 2. What a failure means for the caller + +This is the subtle part, and the reason `PoolRunError` has three variants rather +than being one error type. Each drives different caller behaviour: + +| Variant | The job… | The caller must… | +| --- | --- | --- | +| `PreDispatch` | provably never reached a worker | fall back to a per-call spawn — safe, because nothing ran | +| `PostDispatch` | reached a worker and **may have executed** | **not** retry, or it risks running someone's code twice | +| `Saturated` | was shed because the pool was full | **not** spawn — that reintroduces exactly the resident memory the pool caps. Report busy, or retry later | + +`classify` maps the module's failures onto these. The default is `PreDispatch`, +and the asymmetry is deliberate: mistakenly treating a job as un-run costs one +extra fallback spawn, while mistakenly treating a run job as un-run duplicates +its side effects. Only the two signals the module states explicitly — capacity +and post-dispatch — move a failure out of the default. + +## Key files + +| File | Role | +| --- | --- | +| `mod.rs` | `PoolRunError`, the shared `run_inline` dispatch, `classify`, and `all_stats`. | +| `node.rs` / `python.rs` | Per-language `enabled()` and `run_inline()`; they differ only in which language they name. | +| `types.rs` | `PoolExecOutcome` (with `queue_wait` kept apart from `elapsed`), `PoolLang`, `PoolSettings`. | + +## Notes + +- **`queue_wait` is reported separately from `elapsed` on purpose.** A host that + cannot tell a slow job from a busy pool will tune the wrong knob. +- **`all_stats` returns empty rather than failing** when the module is not + loaded. A status surface wants to render "nothing running", not an error. +- **A worker is still a child of this process.** A TinyBus module is a `cdylib` + loaded in-process, so the resident cost and the process-tree shape the + `library-profile skill-run` gate asserts on are unchanged by the move. From 96289755f24a2425783f5ad6dde94f863f3e718d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 23:24:49 +0300 Subject: [PATCH 38/55] chore(deps): update Cargo.lock Updated the Cargo.lock file to reflect changes in dependencies, ensuring the lockfile remains in sync with the current Cargo.toml. Auto-committed-on: dragonfly Co-authored-by: Medulla --- app/src-tauri/Cargo.lock | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 08a177d746..27991c29c1 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -3489,17 +3489,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" -[[package]] -name = "lzma-sys" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - [[package]] name = "mac-notification-sys" version = "0.6.15" @@ -4434,6 +4423,7 @@ dependencies = [ "tinymemory-core", "tinymemory-tinycortex", "tinyplace", + "tinyruntime-bus", "tinywallet", "tokio", "tokio-stream", @@ -4451,7 +4441,6 @@ dependencies = [ "walkdir", "windows-sys 0.61.2", "x25519-dalek", - "xz2", "zeroize", "zip 2.4.2", ] @@ -7336,6 +7325,14 @@ dependencies = [ "x25519-dalek", ] +[[package]] +name = "tinyruntime-bus" +version = "0.2.1" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -9148,15 +9145,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9cc00251562a284751c9973bace760d86c0276c471b4be569fe6b068ee97a56" -[[package]] -name = "xz2" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] - [[package]] name = "yoke" version = "0.8.3" From 71c2e0c8e8b11dd27c5d47e81e1f5b585e89afa3 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 23:31:53 +0300 Subject: [PATCH 39/55] refactor(runtime): route runtime lookups through a local client facade Replace direct imports from the module bus with a new `client` module that provides an ungated compilation path. This allows the runtime directory to compile even when the module bus feature is disabled, while keeping the same public API surface for consumers. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/client/disabled.rs | 118 ++++++++++++++++++++++ src/openhuman/runtime/client/mod.rs | 23 +++++ src/openhuman/runtime/mod.rs | 4 +- src/openhuman/runtime/node/bootstrap.rs | 2 +- src/openhuman/runtime/pool/mod.rs | 2 +- src/openhuman/runtime/pool/pool_tests.rs | 2 +- src/openhuman/runtime/python/bootstrap.rs | 2 +- 7 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 src/openhuman/runtime/client/disabled.rs create mode 100644 src/openhuman/runtime/client/mod.rs diff --git a/src/openhuman/runtime/client/disabled.rs b/src/openhuman/runtime/client/disabled.rs new file mode 100644 index 0000000000..d128a18431 --- /dev/null +++ b/src/openhuman/runtime/client/disabled.rs @@ -0,0 +1,118 @@ +//! The `modules`-off stand-in for the runtime client. +//! +//! Mirrors the surface the real client exposes, answering everything with +//! [`RuntimeCallError::Unavailable`]. A build without the module bus has no way +//! to reach the `tinyruntime` module, and that is a runtime fact rather than a +//! failure: the shell skips its `PATH` injection, and the exec tools are not +//! registered, exactly as when the runtime is disabled in configuration. + +use tinyruntime_bus::{ + ExecResponse, Language, LanguagesResponse, PoolStatsResponse, ResolvedRuntime, +}; + +use crate::openhuman::config::Config; + +/// Returned by every call in a build without the module bus. +/// +/// Phrased as a build fact, matching the `runtime-node` stub's convention. +const MODULES_DISABLED_MESSAGE: &str = + "the modules feature is disabled at compile time — rebuild with `--features modules` to use \ + managed language runtimes"; + +/// Why a runtime call did not produce what was asked for. +/// +/// The same three variants the real client uses, so callers match identically in +/// both builds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RuntimeCallError { + /// The module is not loaded and cannot be. + Unavailable(String), + /// The request was rejected. + InvalidRequest(String), + /// Resolution, installation, or execution failed. + Failed(String), +} + +impl std::fmt::Display for RuntimeCallError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Unavailable(message) + | Self::InvalidRequest(message) + | Self::Failed(message) => f.write_str(message), + } + } +} + +impl std::error::Error for RuntimeCallError {} + +/// The failure every call in this build produces. +fn unavailable() -> RuntimeCallError { + RuntimeCallError::Unavailable(MODULES_DISABLED_MESSAGE.to_string()) +} + +/// Always unavailable: there is no module bus to ask. +/// +/// # Errors +/// +/// Always [`RuntimeCallError::Unavailable`]. +pub async fn resolve( + _config: &Config, + _language: &Language, + _install: bool, +) -> Result, RuntimeCallError> { + Err(unavailable()) +} + +/// Always unavailable: there is no module bus to ask. +/// +/// # Errors +/// +/// Always [`RuntimeCallError::Unavailable`]. +pub async fn execute( + _config: &Config, + _language: &Language, + _code: impl Into, + _cwd: Option, + _timeout: Option, +) -> Result { + Err(unavailable()) +} + +/// Always unavailable: there is no module bus to ask. +/// +/// # Errors +/// +/// Always [`RuntimeCallError::Unavailable`]. +pub async fn languages(_config: &Config) -> Result { + Err(unavailable()) +} + +/// Always unavailable: there is no module bus to ask. +/// +/// # Errors +/// +/// Always [`RuntimeCallError::Unavailable`]. +pub async fn pool_stats(_config: &Config) -> Result { + Err(unavailable()) +} + +#[cfg(test)] +mod test { + use super::{RuntimeCallError, resolve, unavailable}; + use crate::openhuman::config::Config; + use tinyruntime_bus::Language; + + #[tokio::test] + async fn every_call_reports_the_missing_feature() { + let error = resolve(&Config::default(), &Language::nodejs(), true) + .await + .expect_err("there is no module bus in this build"); + assert!(matches!(error, RuntimeCallError::Unavailable(_))); + assert!(error.to_string().contains("modules feature"), "got `{error}`"); + } + + #[test] + fn the_failure_renders_as_its_message_alone() { + assert_eq!(unavailable().to_string(), super::MODULES_DISABLED_MESSAGE); + } +} diff --git a/src/openhuman/runtime/client/mod.rs b/src/openhuman/runtime/client/mod.rs new file mode 100644 index 0000000000..fcefc12588 --- /dev/null +++ b/src/openhuman/runtime/client/mod.rs @@ -0,0 +1,23 @@ +//! The one way the runtime clients reach the `tinyruntime` module. +//! +//! `openhuman::modules` is behind the `modules` feature, but this directory is +//! not: `ShellTool` holds an `Option>` as a field and is +//! kernel, so the toolchain clients are always compiled. Importing +//! `modules::runtime` directly would therefore break every build with the gate +//! off. +//! +//! So the import goes through here. With `modules` on this is the real client; +//! with it off it is [`disabled`], which answers every call with the same +//! "unavailable" shape a missing module produces. Callers cannot tell the +//! difference, and neither can the shell — which is the point: an off-state must +//! look like a runtime that is simply not there, not like a compile error. + +#[cfg(not(feature = "modules"))] +mod disabled; + +#[cfg(feature = "modules")] +pub(crate) use crate::openhuman::modules::runtime::{ + RuntimeCallError, execute, languages, pool_stats, resolve, +}; +#[cfg(not(feature = "modules"))] +pub(crate) use disabled::{RuntimeCallError, execute, languages, pool_stats, resolve}; diff --git a/src/openhuman/runtime/mod.rs b/src/openhuman/runtime/mod.rs index 86dae216eb..7df7a2ae86 100644 --- a/src/openhuman/runtime/mod.rs +++ b/src/openhuman/runtime/mod.rs @@ -4,7 +4,8 @@ //! is *here* is the client half — everything that downloads a toolchain, //! verifies it, unpacks it, caches it, or keeps a warm worker in front of it now //! lives in the `tinyruntime` module, behind -//! [`crate::openhuman::modules::runtime`]. +//! [`crate::openhuman::modules::runtime`], reached through the ungated +//! [`client`] facade so a build without the module bus still compiles. //! //! That split is what this directory is for: adapting module answers onto the //! types the rest of the core already names, so a migration of the machinery did @@ -21,6 +22,7 @@ //! machinery: a client that asks a module for a path needs neither a //! decompressor nor a download pipeline. +pub mod client; pub mod javascript; pub mod node; pub mod pool; diff --git a/src/openhuman/runtime/node/bootstrap.rs b/src/openhuman/runtime/node/bootstrap.rs index c90a3f2743..804a9b1779 100644 --- a/src/openhuman/runtime/node/bootstrap.rs +++ b/src/openhuman/runtime/node/bootstrap.rs @@ -30,7 +30,7 @@ use anyhow::{anyhow, Result}; use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; use crate::openhuman::config::Config; -use crate::openhuman::modules::runtime; +use crate::openhuman::runtime::client as runtime; /// Origin of the resolved toolchain — feeds into logging and lets the caller /// decide whether to surface a "Node was downloaded to …" message in the UI. diff --git a/src/openhuman/runtime/pool/mod.rs b/src/openhuman/runtime/pool/mod.rs index d6f41b94c5..cd2e507b67 100644 --- a/src/openhuman/runtime/pool/mod.rs +++ b/src/openhuman/runtime/pool/mod.rs @@ -29,7 +29,7 @@ pub mod types; use tinyruntime_bus::Language; use crate::openhuman::config::Config; -use crate::openhuman::modules::runtime::{self, RuntimeCallError}; +use crate::openhuman::runtime::client::{self as runtime, RuntimeCallError}; pub use types::{PoolExecOutcome, PoolLang, PoolSettings}; diff --git a/src/openhuman/runtime/pool/pool_tests.rs b/src/openhuman/runtime/pool/pool_tests.rs index 52bb00b7f0..ff17ccec1c 100644 --- a/src/openhuman/runtime/pool/pool_tests.rs +++ b/src/openhuman/runtime/pool/pool_tests.rs @@ -7,7 +7,7 @@ use super::{classify, node, python, PoolRunError}; use crate::openhuman::config::Config; -use crate::openhuman::modules::runtime::RuntimeCallError; +use crate::openhuman::runtime::client::RuntimeCallError; #[test] fn node_pools_by_default_and_python_does_not() { diff --git a/src/openhuman/runtime/python/bootstrap.rs b/src/openhuman/runtime/python/bootstrap.rs index 04b1bc0f24..f1c6731e1c 100644 --- a/src/openhuman/runtime/python/bootstrap.rs +++ b/src/openhuman/runtime/python/bootstrap.rs @@ -21,7 +21,7 @@ use anyhow::{anyhow, Result}; use tinyruntime_bus::{Language, ResolvedRuntime, RuntimeSource}; use crate::openhuman::config::Config; -use crate::openhuman::modules::runtime; +use crate::openhuman::runtime::client as runtime; /// Origin of the resolved interpreter. #[derive(Debug, Clone, Copy, PartialEq, Eq)] From 9fa8effc8ccf14f6e73a746eedb2d502d147606c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 23:38:21 +0300 Subject: [PATCH 40/55] chore: files changed src/openhuman/runtime/client/disabled.rs,src/openhuman/runtime/client/mod.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/client/disabled.rs | 13 ++++++++----- src/openhuman/runtime/client/mod.rs | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/openhuman/runtime/client/disabled.rs b/src/openhuman/runtime/client/disabled.rs index d128a18431..53d49be423 100644 --- a/src/openhuman/runtime/client/disabled.rs +++ b/src/openhuman/runtime/client/disabled.rs @@ -36,9 +36,9 @@ pub enum RuntimeCallError { impl std::fmt::Display for RuntimeCallError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - Self::Unavailable(message) - | Self::InvalidRequest(message) - | Self::Failed(message) => f.write_str(message), + Self::Unavailable(message) | Self::InvalidRequest(message) | Self::Failed(message) => { + f.write_str(message) + } } } } @@ -98,7 +98,7 @@ pub async fn pool_stats(_config: &Config) -> Result Date: Fri, 21 Aug 2026 23:54:57 +0300 Subject: [PATCH 41/55] chore(runtime): remove unused languages function from disabled client The `languages` function in the disabled runtime client was removed because it is no longer called anywhere in the codebase, and its associated `LanguagesResponse` import was cleaned up to keep the module minimal. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/client/disabled.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/openhuman/runtime/client/disabled.rs b/src/openhuman/runtime/client/disabled.rs index 53d49be423..da00a5da94 100644 --- a/src/openhuman/runtime/client/disabled.rs +++ b/src/openhuman/runtime/client/disabled.rs @@ -6,9 +6,7 @@ //! failure: the shell skips its `PATH` injection, and the exec tools are not //! registered, exactly as when the runtime is disabled in configuration. -use tinyruntime_bus::{ - ExecResponse, Language, LanguagesResponse, PoolStatsResponse, ResolvedRuntime, -}; +use tinyruntime_bus::{ExecResponse, Language, PoolStatsResponse, ResolvedRuntime}; use crate::openhuman::config::Config; @@ -78,15 +76,6 @@ pub async fn execute( Err(unavailable()) } -/// Always unavailable: there is no module bus to ask. -/// -/// # Errors -/// -/// Always [`RuntimeCallError::Unavailable`]. -pub async fn languages(_config: &Config) -> Result { - Err(unavailable()) -} - /// Always unavailable: there is no module bus to ask. /// /// # Errors From 33eba717161909deb596438bbb800922df770d7a Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Fri, 21 Aug 2026 23:57:17 +0300 Subject: [PATCH 42/55] chore(client): drop unused `languages` re-export from the modules facade The `languages` function was re-exported from the `modules::runtime` facade but is never called by anything in the `runtime/` crate, so it has been removed from both the real and the disabled-stub re-export lists to keep the public surface minimal and avoid forcing the stub to maintain an unused twin. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/client/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/openhuman/runtime/client/mod.rs b/src/openhuman/runtime/client/mod.rs index d09848e6d8..24fca2db45 100644 --- a/src/openhuman/runtime/client/mod.rs +++ b/src/openhuman/runtime/client/mod.rs @@ -15,9 +15,12 @@ #[cfg(not(feature = "modules"))] mod disabled; +// Only what `runtime/` actually calls. `modules::runtime` exposes more — the +// `Languages` listing, for one — but a facade that re-exported the whole surface +// would oblige the stub to grow a twin of every member nothing here uses. #[cfg(feature = "modules")] pub(crate) use crate::openhuman::modules::runtime::{ - execute, languages, pool_stats, resolve, RuntimeCallError, + execute, pool_stats, resolve, RuntimeCallError, }; #[cfg(not(feature = "modules"))] -pub(crate) use disabled::{execute, languages, pool_stats, resolve, RuntimeCallError}; +pub(crate) use disabled::{execute, pool_stats, resolve, RuntimeCallError}; From 7edcb940d1a3e096c4beb769a352b6958713d185 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 00:12:20 +0300 Subject: [PATCH 43/55] chore(scripts): update kernel-floor limits for tinyruntime-bus addition The kernel-floor limits file is updated to reflect the addition of the tinyruntime-bus package, which moves language runtimes behind the TinyBus module. This change increases the package count from 283 to 284 and the name count from 265 to 266 while keeping native dependencies unchanged. Auto-committed-on: dragonfly Co-authored-by: Medulla --- scripts/kernel-floor.limits | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/kernel-floor.limits b/scripts/kernel-floor.limits index e4df41a888..053b381349 100644 --- a/scripts/kernel-floor.limits +++ b/scripts/kernel-floor.limits @@ -13,6 +13,37 @@ # Simulate with: scripts/dep-sim.py --cut # # History +# 284/266/2 2026-08-22 language runtimes moved behind the tinyruntime +# TinyBus module (+1 package, +1 NAME, native unchanged). +# It is a RAISE, so: the name is `tinyruntime-bus`, the +# wire contract for the runtime router. It lands on the +# always-on path and cannot be gated. `ShellTool` holds an +# `Option>` as a FIELD and +# `tools/impl/system/shell.rs` is kernel, so the toolchain +# clients in `runtime/` are always compiled — the same +# constraint that already forces `runtime/node/stub.rs` to +# exist — and those clients name the contract's payload +# types. Gating them would mean a parallel set of runtime +# types for the gates-off build, which is worse than one +# small crate. +# No new third-party code enters. `tinyruntime-bus` is +# deliberately dependency-light — `serde` and `serde_json`, +# both already in this profile, and nothing else; its own +# CI asserts it stays free of a transport, an async +# runtime, an HTTP client and any native library. The +# diff is exactly one name in and none out: `cargo tree +# --no-default-features --features flows -e normal` +# before/after differ by `tinyruntime-bus` alone. +# What it buys is a much larger shed elsewhere, which this +# profile cannot see: downloading, verifying and unpacking +# language toolchains left this repository entirely, taking +# `xz2` AND its static liblzma C build with it. The +# 2026-08-09 entry below shed those from the `flows` +# profile by GATING them behind `runtime-node`; they are +# now gone from every configuration, including the product +# set that ships. `runtime-node = []` as a result. +# Measured on Linux: `scripts/kernel-floor.sh flows` -> +# 284/266/2, against 283/265/2 at this branch's base. # 283/265/2 2026-08-21 tinymemory bumped past its #76/#77 line (-2 packages, # +1 NAME). The name is `tinymemory-bus`, and it is a # RAISE, so: tinymemory#74 moved the wire vocabulary — @@ -371,4 +402,4 @@ # (libsqlite3-sys, ring) — see docs/plans MIGRATION-PLAN G6. # 307/284 2026-08-12 Re-baseline after the upstream lockfile resolution; # `flows` remains at two native packages. -flows:283:265:2 +flows:284:266:2 From d7ce6197daf5bdb7a7271d54a184378b1b97eadf Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 00:39:01 +0300 Subject: [PATCH 44/55] chore(ci): update expected name count in dep-sim guard The expected name count for the `dep-sim.py` guard in the CI lite workflow is raised from 265 to 266 to reflect the addition of the `tinyruntime-bus` name, which was introduced when language runtimes were moved behind the TinyBus module. The comment is also updated to document the reason for the change and to point to the matching entry in the kernel floor limits file. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .github/workflows/ci-lite.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-lite.yml b/.github/workflows/ci-lite.yml index 0477b2d743..779b0ff684 100644 --- a/.github/workflows/ci-lite.yml +++ b/.github/workflows/ci-lite.yml @@ -561,10 +561,15 @@ jobs: # # The number is the Linux name count of the `flows` profile, so it moves # whenever scripts/kernel-floor.limits does and belongs in the same PR. - # 264 -> 265 on 2026-08-21: the tinymemory #76/#77 bump adds exactly one - # name, `tinymemory-bus`. macOS resolves one higher (266) per the host - # skew recorded in the limits history — this expects the CI host. - run: python3 scripts/dep-sim.py --cut-nothing --expect-names 265 + # 265 -> 266 on 2026-08-22: language runtimes moved behind the + # tinyruntime TinyBus module, which puts exactly one name on the + # always-on path — `tinyruntime-bus`, the wire contract the toolchain + # clients name and which cannot be gated because `ShellTool` is kernel. + # See the matching entry in scripts/kernel-floor.limits for why, and for + # what it buys (xz2 and its liblzma C build leave every configuration). + # macOS resolves one higher per the host skew recorded there — this + # expects the CI host. + run: python3 scripts/dep-sim.py --cut-nothing --expect-names 266 - name: Guard — new feature-gated test modules must be acknowledged # Self-maintaining coverage: the set of source files that #[cfg]-gate a test on From 232d274c76ef3449de635f46ed36fcd89a6ccfb7 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 00:51:01 +0300 Subject: [PATCH 45/55] chore(deps): update tinyagents subproject commit Updated the pinned commit for the tinyagents vendored dependency to incorporate upstream changes. Auto-committed-on: dragonfly Co-authored-by: Medulla --- vendor/tinyagents | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vendor/tinyagents b/vendor/tinyagents index 65c02fbf93..f36b1825f2 160000 --- a/vendor/tinyagents +++ b/vendor/tinyagents @@ -1 +1 @@ -Subproject commit 65c02fbf9392f736a4ee169870e36e702dc5d923 +Subproject commit f36b1825f2345e1d1c69efae9ca00ec8f8ca1af7 From 75e22fee47e4e03448e325c04839cef04457d0f0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 00:52:08 +0300 Subject: [PATCH 46/55] fix(AGENTS.md): correct native build count in contributor feature set The contributor feature set now lists two native builds instead of three, reflecting the removal of `lzma-sys` from the default features. Auto-committed-on: dragonfly Co-authored-by: Medulla --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 179e447572..b42f2287c9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -412,7 +412,7 @@ Per-domain Cargo features drop whole domains **at compile time** (smaller binary | Set | Where it lives | What it is | | --- | --- | --- | -| **Contributor** | `[features] default` in `Cargo.toml` | What a bare `cargo check`, `cargo test` and rust-analyzer compile. 10 cheap gates. **~353 packages / 3 native builds** (`libsqlite3-sys`, `lzma-sys`, `ring`). | +| **Contributor** | `[features] default` in `Cargo.toml` | What a bare `cargo check`, `cargo test` and rust-analyzer compile. 10 cheap gates. **~353 packages / 2 native builds** (`libsqlite3-sys`, `ring`). | > **`modules` is in `default`, and it is the one gate here that is not optional.** > The table below has documented it as Contrib=ON since it landed and From db2d844b92eb4c1095f078763d37e658c6c52696 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 00:52:21 +0300 Subject: [PATCH 47/55] fix(runtime): implement custom Debug for NodeBootstrap stub The stub's Debug implementation now mirrors the real client by redacting the Config field, ensuring both render identically in logs and preventing the disabled build from leaking secrets like api_key. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/runtime/node/stub.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/openhuman/runtime/node/stub.rs b/src/openhuman/runtime/node/stub.rs index 1656ed7131..4651127238 100644 --- a/src/openhuman/runtime/node/stub.rs +++ b/src/openhuman/runtime/node/stub.rs @@ -61,11 +61,22 @@ pub struct ResolvedNode { } /// Inert stand-in for the toolchain client. -#[derive(Debug)] pub struct NodeBootstrap { config: Arc, } +impl std::fmt::Debug for NodeBootstrap { + /// The real client redacts `Config` because it is full of secrets; the stub + /// mirrors that so the two render identically in logs, rather than the + /// disabled build being the one that leaks an `api_key` into a debug line. + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("NodeBootstrap") + .field("resolved", &false) + .finish_non_exhaustive() + } +} + impl NodeBootstrap { /// Build a stub over this host's configuration. /// From f1b9ef0e717a6f1464793cbbc94b54285605b21f Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 02:16:29 +0300 Subject: [PATCH 48/55] test(raw_coverage): simplify NodeBootstrap construction in e2e test Updated the test helper to pass a single Arc-wrapped config clone instead of three separate arguments when constructing NodeBootstrap, matching a recent refactor of the production API and reducing boilerplate in the test setup. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tools_approval_channels_raw_coverage_e2e.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs index e3e0d1772b..14b50461ab 100644 --- a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs +++ b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs @@ -3535,11 +3535,7 @@ async fn node_and_npm_exec_tools_cover_validation_policy_and_disabled_runtime_pa &config.workspace_dir, )); let runtime = Arc::new(NativeRuntime::new()); - let bootstrap = Arc::new(NodeBootstrap::new( - config.node.clone(), - workspace, - reqwest::Client::new(), - )); + let bootstrap = Arc::new(NodeBootstrap::new(Arc::new(config.clone()))); let node = NodeExecTool::new( full_security.clone(), From c2c418bd68e01e1b18903c7b1db0227d3791d93d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 02:22:02 +0300 Subject: [PATCH 49/55] test(Feedback): add mock for validateFeedback in test setup Add a mock for the validateFeedback API function in the Feedback test file, including a default resolved value, to support upcoming tests that exercise feedback validation logic. Auto-committed-on: dragonfly Co-authored-by: Medulla --- app/src/pages/Feedback.test.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/src/pages/Feedback.test.tsx b/app/src/pages/Feedback.test.tsx index 14b08b3c6a..4237b63fa7 100644 --- a/app/src/pages/Feedback.test.tsx +++ b/app/src/pages/Feedback.test.tsx @@ -12,6 +12,7 @@ const mockSubmit = vi.fn(); const mockUpdateStatus = vi.fn(); const mockGetFeedback = vi.fn(); const mockAddComment = vi.fn(); +const mockValidateFeedback = vi.fn(); vi.mock('../services/api/feedbackApi', () => ({ feedbackApi: { @@ -21,6 +22,7 @@ vi.mock('../services/api/feedbackApi', () => ({ updateStatus: (...args: unknown[]) => mockUpdateStatus(...args), getFeedback: (...args: unknown[]) => mockGetFeedback(...args), addComment: (...args: unknown[]) => mockAddComment(...args), + validateFeedback: (...args: unknown[]) => mockValidateFeedback(...args), }, })); @@ -66,6 +68,8 @@ describe('', () => { mockUpdateStatus.mockReset(); mockGetFeedback.mockReset(); mockAddComment.mockReset(); + mockValidateFeedback.mockReset(); + mockValidateFeedback.mockResolvedValue({ tier: 'pass', reason: 'ok' }); userRole.current = 'user'; }); @@ -113,6 +117,8 @@ describe(' keeps the board in sync after local mutations', () => { mockUpdateStatus.mockReset(); mockGetFeedback.mockReset(); mockAddComment.mockReset(); + mockValidateFeedback.mockReset(); + mockValidateFeedback.mockResolvedValue({ tier: 'pass', reason: 'ok' }); userRole.current = 'user'; }); From 09afdac5b37bf5a3ee7b040680169f68e6150d70 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 02:30:41 +0300 Subject: [PATCH 50/55] fix(tools): share config Arc instead of cloning root_config The bootstrap tools now receive an Arc::clone of the session's config reference rather than a separately-cloned root_config, ensuring that the registry and all language clients operate on the same configuration snapshot for the entire session. Auto-committed-on: dragonfly Co-authored-by: Medulla --- src/openhuman/tools/ops.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/openhuman/tools/ops.rs b/src/openhuman/tools/ops.rs index 6678104181..e491378e04 100644 --- a/src/openhuman/tools/ops.rs +++ b/src/openhuman/tools/ops.rs @@ -112,8 +112,11 @@ pub fn all_tools_with_runtime( // One shared snapshot of this session's configuration for both language // clients. They each hand it to the `tinyruntime` module on every call — // the module holds no configuration of its own — so the two must not be - // able to disagree about which version this session asked for. - let shared_config = Arc::new(root_config.clone()); + // able to disagree about which version this session asked for. The + // registry is assembled under `config`, so the bootstraps share that same + // Arc rather than a separately-cloned `root_config` — one configuration + // snapshot for everything this session builds. + let shared_config = Arc::clone(&config); // Build a session-scoped managed Node.js bootstrap once, so ShellTool, // NodeExecTool, and NpmExecTool all share the same memoised resolution From afa1a5560422d19bcc4c9b65f28865c31789fb00 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 11:30:07 +0300 Subject: [PATCH 51/55] test(raw_coverage): add missing tool_specs field to test fixture The test fixture for the debug dump writer was missing the `tool_specs` field, which caused a compilation error after the struct was extended. This change adds an empty vector to satisfy the new field requirement. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../agent_archivist_debug_round21_raw_coverage_e2e.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs index 281edb9999..3b71a0214f 100644 --- a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs @@ -436,6 +436,7 @@ fn debug_dump_writer_sanitizes_names_and_writes_summary_sidecars() -> Result<()> workspace_dir: PathBuf::from("/tmp/round21-workspace"), text: "SYSTEM PROMPT\n".to_string(), tool_names: vec!["echo".to_string(), "search".to_string()], + tool_specs: vec![], skill_tool_count: 1, }]; From 7cc661e82dc664ab14b46da45b4832ec7b002423 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 11:30:10 +0300 Subject: [PATCH 52/55] test(raw_coverage): add missing tool_specs field to expected prompt dumps The test assertions for agent debug prompt dumps were missing the tool_specs field in the expected DumpedPrompt structs, causing compilation failures after the struct was extended. This change adds empty tool_specs vectors to match the updated struct definition. Auto-committed-on: dragonfly Co-authored-by: Medulla --- tests/raw_coverage/inference_agent_raw_coverage_e2e.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs index a936248543..ce636e219b 100644 --- a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs +++ b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs @@ -3875,6 +3875,7 @@ async fn agent_debug_prompt_dump_and_identity_rendering_cover_file_layouts() { workspace_dir: workspace.path().join("ws"), text: "# planner\nbody\n".to_string(), tool_names: vec!["todo".to_string(), "delegate".to_string()], + tool_specs: vec![], skill_tool_count: 0, }, DumpedPrompt { @@ -3885,6 +3886,7 @@ async fn agent_debug_prompt_dump_and_identity_rendering_cover_file_layouts() { workspace_dir: workspace.path().join("ws"), text: "# integrations\nbody\n".to_string(), tool_names: vec!["GMAIL_SEND_EMAIL".to_string()], + tool_specs: vec![], skill_tool_count: 1, }, ]; From 9cc8817d7e7f0966867b644bb12e61f7af3fccad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 12:34:35 +0300 Subject: [PATCH 53/55] test(raw_coverage): update assertion for delegate-tool description The test assertion for the research tool's description is updated to reflect that the repeated "direct tools are insufficient" prefix has been intentionally removed from delegate-tool descriptions, as the orchestrator's own prompt already carries that rule once. The assertion now checks for the remaining content that confirms the description still contains the target agent's `when_to_use` guidance. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tools_approval_channels_raw_coverage_e2e.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs index 14b50461ab..b5c3bf0b7d 100644 --- a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs +++ b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs @@ -1620,12 +1620,16 @@ async fn orchestrator_tool_synthesis_covers_agent_and_integration_delegation_edg assert_eq!(names, vec!["research", "delegate_to_integrations_agent"]); let research = &tools[0]; + // Delegate-tool descriptions carry the target agent's `when_to_use` + // verbatim. The repeated "direct response/direct tools are insufficient" + // prefix was intentionally removed (the orchestrator's own prompt already + // carries that rule once), so assert it is gone rather than still required. assert!(research .description() - .contains("direct tools are insufficient")); + .contains("careful public-source research")); assert!(research .description() - .contains("careful public-source research")); + .contains("route")); assert_eq!(research.permission_level(), PermissionLevel::Execute); assert_eq!(research.category(), ToolCategory::System); assert_eq!( From 77a81c057f48e737503c9be7442702a173648f21 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 12:35:37 +0300 Subject: [PATCH 54/55] fix(tests): update research description assertion in e2e coverage test The test assertion for the research tool's description was updated to match the new expected text, and an additional assertion was added to verify that the description no longer contains a phrase about direct tools being insufficient. This ensures the test reflects the current behaviour of the approval channels raw coverage. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../raw_coverage/tools_approval_channels_raw_coverage_e2e.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs index b5c3bf0b7d..449cfbe6cf 100644 --- a/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs +++ b/tests/raw_coverage/tools_approval_channels_raw_coverage_e2e.rs @@ -1629,7 +1629,10 @@ async fn orchestrator_tool_synthesis_covers_agent_and_integration_delegation_edg .contains("careful public-source research")); assert!(research .description() - .contains("route")); + .contains("Use for careful public-source research.")); + assert!(!research + .description() + .contains("direct tools are insufficient")); assert_eq!(research.permission_level(), PermissionLevel::Execute); assert_eq!(research.category(), ToolCategory::System); assert_eq!( From 4359399c5fa5e56348837ed980aa9a0f5bad8815 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sat, 22 Aug 2026 13:08:12 +0300 Subject: [PATCH 55/55] fix(tests): remove duplicate tool_specs fields in raw coverage test structs Two raw coverage end-to-end test files contained struct literals with duplicate `tool_specs` fields, which would cause a compilation error in strict Rust editions. The extra, redundant fields have been removed to keep the test data valid and compile cleanly. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../agent_archivist_debug_round21_raw_coverage_e2e.rs | 1 - tests/raw_coverage/inference_agent_raw_coverage_e2e.rs | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs index cbf0c48411..3b71a0214f 100644 --- a/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs +++ b/tests/raw_coverage/agent_archivist_debug_round21_raw_coverage_e2e.rs @@ -438,7 +438,6 @@ fn debug_dump_writer_sanitizes_names_and_writes_summary_sidecars() -> Result<()> tool_names: vec!["echo".to_string(), "search".to_string()], tool_specs: vec![], skill_tool_count: 1, - tool_specs: Vec::new(), }]; let summary = write_prompt_dumps(tmp.path(), &dumps)?; diff --git a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs index 352a99d0dd..ce636e219b 100644 --- a/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs +++ b/tests/raw_coverage/inference_agent_raw_coverage_e2e.rs @@ -3877,7 +3877,6 @@ async fn agent_debug_prompt_dump_and_identity_rendering_cover_file_layouts() { tool_names: vec!["todo".to_string(), "delegate".to_string()], tool_specs: vec![], skill_tool_count: 0, - tool_specs: Vec::new(), }, DumpedPrompt { agent_id: "integrations_agent".to_string(), @@ -3889,7 +3888,6 @@ async fn agent_debug_prompt_dump_and_identity_rendering_cover_file_layouts() { tool_names: vec!["GMAIL_SEND_EMAIL".to_string()], tool_specs: vec![], skill_tool_count: 1, - tool_specs: Vec::new(), }, ];