From 29ae83e92b117e7b1661971fb62e8bd693f7613c Mon Sep 17 00:00:00 2001 From: Enrico Piovesan Date: Mon, 27 Jul 2026 21:19:40 -0600 Subject: [PATCH 1/3] feat: harden datastore ownership lock --- crates/traverse-runtime/src/data_store.rs | 151 ++++++++++++++++-- ...19-embedder-owned-datastore-integration.md | 2 + 2 files changed, 143 insertions(+), 10 deletions(-) diff --git a/crates/traverse-runtime/src/data_store.rs b/crates/traverse-runtime/src/data_store.rs index c136782a..26ff5a71 100644 --- a/crates/traverse-runtime/src/data_store.rs +++ b/crates/traverse-runtime/src/data_store.rs @@ -298,9 +298,7 @@ impl LocalFileDataStore { .truncate(false) .open(&lock_path) .map_err(|error| io_error("open data store lock", &error))?; - lock_file - .try_lock() - .map_err(|error| lock_error(error, &root))?; + lock_file.try_lock().map_err(lock_error)?; Ok(Self { root, classification, @@ -417,14 +415,25 @@ fn digest_for_record(record: &StateRecord) -> Result { Ok(format!("sha256:{hexadecimal}")) } -fn lock_error(error: TryLockError, root: &PathBuf) -> DataStoreError { +fn lock_error(error: TryLockError) -> DataStoreError { match error { TryLockError::WouldBlock => data_store_error( DataStoreErrorCode::StoreLocked, "store_locked", - json!({ "root": root }), + json!({ "reason": "exclusive_owner_active" }), + ), + TryLockError::Error(error) if error.kind() == std::io::ErrorKind::Unsupported => { + data_store_error( + DataStoreErrorCode::IoFailure, + "storage_io_failed", + json!({ "operation": "acquire_lock", "reason": "locking_unsupported" }), + ) + } + TryLockError::Error(_) => data_store_error( + DataStoreErrorCode::IoFailure, + "storage_io_failed", + json!({ "operation": "acquire_lock", "reason": "lock_acquisition_failed" }), ), - TryLockError::Error(error) => io_error("lock data store root", &error), } } @@ -647,6 +656,9 @@ mod tests { use super::*; use serde_json::json; use std::cell::Cell; + use std::process::{Child, Command, Stdio}; + use std::thread; + use std::time::Duration; use traverse_contracts::{ BinaryFormat, CapabilityContract, Condition, DependencyReference, Entrypoint, EntrypointKind, EventReference, Execution, ExecutionConstraints, ExecutionTarget, @@ -1070,6 +1082,80 @@ mod tests { let second_owner = LocalFileDataStore::new(&root).expect_err("second owner must fail"); assert_eq!(second_owner.code, DataStoreErrorCode::StoreLocked); assert_eq!(second_owner.message, "store_locked"); + assert_eq!( + second_owner.details, + json!({ "reason": "exclusive_owner_active" }) + ); + } + + #[test] + fn local_file_adapter_lock_child() { + let Ok(root) = std::env::var("TRAVERSE_DATA_STORE_LOCK_CHILD_ROOT") else { + return; + }; + let root = PathBuf::from(root); + let _adapter = LocalFileDataStore::new(&root).expect("child should acquire lock"); + fs::write(lock_child_ready_path(&root), "ready").expect("child should signal readiness"); + for _ in 0..500 { + if lock_child_release_path(&root).exists() { + return; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("parent did not release the lock child"); + } + + #[test] + fn local_file_adapter_rejects_cross_process_owner_and_recovers_after_exit() { + let root = temp_root("cross-process-lock"); + let mut initial_owner = LocalFileDataStore::new(&root).expect("initial owner should open"); + let committed = record("draft", "writer-a", 1, json!("committed")); + initial_owner + .write(committed.clone()) + .expect("initial write should succeed"); + drop(initial_owner); + + let mut child = start_lock_child(&root); + wait_for_lock_child(&root); + let blocked = LocalFileDataStore::new(&root).expect_err("second process must be blocked"); + assert_eq!(blocked.code, DataStoreErrorCode::StoreLocked); + assert_eq!( + blocked.details, + json!({ "reason": "exclusive_owner_active" }) + ); + + fs::write(lock_child_release_path(&root), "release").expect("parent should release child"); + assert!(child.wait().expect("child should exit").success()); + let reopened = LocalFileDataStore::new(&root).expect("released lock should reopen"); + assert_eq!( + reopened + .read("draft") + .expect("committed record should remain readable"), + Some(committed) + ); + } + + #[test] + fn local_file_adapter_recovers_after_lock_owner_crash() { + let root = temp_root("owner-crash-lock"); + let mut initial_owner = LocalFileDataStore::new(&root).expect("initial owner should open"); + initial_owner + .write(record("draft", "writer-a", 1, json!("committed"))) + .expect("initial write should succeed"); + drop(initial_owner); + + let mut child = start_lock_child(&root); + wait_for_lock_child(&root); + child.kill().expect("parent should terminate child"); + child.wait().expect("terminated child should exit"); + + let reopened = LocalFileDataStore::new(&root).expect("crashed owner lock should release"); + assert_eq!( + reopened + .read("draft") + .expect("committed record should remain readable"), + Some(record("draft", "writer-a", 1, json!("committed"))) + ); } #[test] @@ -1099,12 +1185,25 @@ mod tests { ); assert_eq!(unknown_version.details["reason"], "unknown_format_version"); - let lock_io = lock_error( - TryLockError::Error(std::io::Error::other("lock device failure")), - &root, - ); + let lock_io = lock_error(TryLockError::Error(std::io::Error::other( + "lock device failure", + ))); assert_eq!(lock_io.code, DataStoreErrorCode::IoFailure); assert_eq!(lock_io.message, "storage_io_failed"); + assert_eq!( + lock_io.details, + json!({ "operation": "acquire_lock", "reason": "lock_acquisition_failed" }) + ); + + let unsupported_lock = lock_error(TryLockError::Error(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "locking unavailable", + ))); + assert_eq!(unsupported_lock.code, DataStoreErrorCode::IoFailure); + assert_eq!( + unsupported_lock.details, + json!({ "operation": "acquire_lock", "reason": "locking_unsupported" }) + ); let durability = durability_error( &root, @@ -1139,6 +1238,38 @@ mod tests { std::env::temp_dir().join(format!("traverse-data-store-{name}-{}", Uuid::new_v4())) } + fn lock_child_ready_path(root: &PathBuf) -> PathBuf { + root.join(".lock-child-ready") + } + + fn lock_child_release_path(root: &PathBuf) -> PathBuf { + root.join(".lock-child-release") + } + + fn start_lock_child(root: &PathBuf) -> Child { + Command::new(std::env::current_exe().expect("test binary path should resolve")) + .args([ + "--exact", + "data_store::tests::local_file_adapter_lock_child", + "--nocapture", + ]) + .env("TRAVERSE_DATA_STORE_LOCK_CHILD_ROOT", root) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("lock child should start") + } + + fn wait_for_lock_child(root: &PathBuf) { + for _ in 0..500 { + if lock_child_ready_path(root).exists() { + return; + } + thread::sleep(Duration::from_millis(10)); + } + panic!("lock child did not become ready"); + } + fn stateful_contract(state_schema: Option) -> CapabilityContract { CapabilityContract { kind: "capability_contract".to_string(), diff --git a/docs/adr/0019-embedder-owned-datastore-integration.md b/docs/adr/0019-embedder-owned-datastore-integration.md index b44cfefa..c66735bd 100644 --- a/docs/adr/0019-embedder-owned-datastore-integration.md +++ b/docs/adr/0019-embedder-owned-datastore-integration.md @@ -15,6 +15,8 @@ Embedder hosts explicitly construct and inject one DataStore for one host-select The store has one fixed public or private classification, one active owner, and stable typed failures. A second owner receives `store_locked`; no lease, concurrent writer, transaction, scan, migration, retention, backup, encryption, sync, or whole-store lifecycle behavior is introduced. Legacy and unknown persisted formats remain fail-closed. State telemetry contains only safe operation metadata. +The baseline lock uses Rust's standard-library non-blocking file lock on the store-local lock file. Supported native hosts are Rust targets where that operation is implemented by the standard library (Unix and Windows). The owner holds the lock for the lifetime of the adapter; dropping the adapter releases it, and operating-system process termination releases a crashed owner's lock before a later open may succeed. A contending owner receives `store_locked` with only the stable `exclusive_owner_active` reason. A target that reports file locking as unsupported fails deterministically with `storage_io_failed` and reason `locking_unsupported`; no fallback lock protocol is attempted. Other lock-acquisition failures expose only the stable `lock_acquisition_failed` reason. Lock paths, store roots, keys, and values are never emitted in these failures. + The first conformance evidence is a Rust embedder example and restart/integrity integration test. Platform-specific lock lifecycle and crash-recovery evidence must be recorded before broadening the supported-platform claim. ## Consequences From 218c99080fe53aae85d93d3b32330aa5ab9ce6d6 Mon Sep 17 00:00:00 2001 From: Enrico Piovesan Date: Mon, 27 Jul 2026 21:19:40 -0600 Subject: [PATCH 2/3] feat: harden datastore ownership lock --- crates/traverse-runtime/src/data_store.rs | 25 ++++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/traverse-runtime/src/data_store.rs b/crates/traverse-runtime/src/data_store.rs index 26ff5a71..ee017e43 100644 --- a/crates/traverse-runtime/src/data_store.rs +++ b/crates/traverse-runtime/src/data_store.rs @@ -656,6 +656,7 @@ mod tests { use super::*; use serde_json::json; use std::cell::Cell; + use std::path::Path; use std::process::{Child, Command, Stdio}; use std::thread; use std::time::Duration; @@ -1089,20 +1090,20 @@ mod tests { } #[test] - fn local_file_adapter_lock_child() { + fn local_file_adapter_lock_child() -> Result<(), String> { let Ok(root) = std::env::var("TRAVERSE_DATA_STORE_LOCK_CHILD_ROOT") else { - return; + return Ok(()); }; let root = PathBuf::from(root); let _adapter = LocalFileDataStore::new(&root).expect("child should acquire lock"); fs::write(lock_child_ready_path(&root), "ready").expect("child should signal readiness"); for _ in 0..500 { if lock_child_release_path(&root).exists() { - return; + return Ok(()); } thread::sleep(Duration::from_millis(10)); } - panic!("parent did not release the lock child"); + Err("parent did not release the lock child".to_string()) } #[test] @@ -1116,7 +1117,7 @@ mod tests { drop(initial_owner); let mut child = start_lock_child(&root); - wait_for_lock_child(&root); + wait_for_lock_child(&root).expect("lock child should become ready"); let blocked = LocalFileDataStore::new(&root).expect_err("second process must be blocked"); assert_eq!(blocked.code, DataStoreErrorCode::StoreLocked); assert_eq!( @@ -1145,7 +1146,7 @@ mod tests { drop(initial_owner); let mut child = start_lock_child(&root); - wait_for_lock_child(&root); + wait_for_lock_child(&root).expect("lock child should become ready"); child.kill().expect("parent should terminate child"); child.wait().expect("terminated child should exit"); @@ -1238,15 +1239,15 @@ mod tests { std::env::temp_dir().join(format!("traverse-data-store-{name}-{}", Uuid::new_v4())) } - fn lock_child_ready_path(root: &PathBuf) -> PathBuf { + fn lock_child_ready_path(root: &Path) -> PathBuf { root.join(".lock-child-ready") } - fn lock_child_release_path(root: &PathBuf) -> PathBuf { + fn lock_child_release_path(root: &Path) -> PathBuf { root.join(".lock-child-release") } - fn start_lock_child(root: &PathBuf) -> Child { + fn start_lock_child(root: &Path) -> Child { Command::new(std::env::current_exe().expect("test binary path should resolve")) .args([ "--exact", @@ -1260,14 +1261,14 @@ mod tests { .expect("lock child should start") } - fn wait_for_lock_child(root: &PathBuf) { + fn wait_for_lock_child(root: &Path) -> Result<(), &'static str> { for _ in 0..500 { if lock_child_ready_path(root).exists() { - return; + return Ok(()); } thread::sleep(Duration::from_millis(10)); } - panic!("lock child did not become ready"); + Err("lock child did not become ready") } fn stateful_contract(state_schema: Option) -> CapabilityContract { From 8fb70d63d869fffb664c4960d4bd9226619cd318 Mon Sep 17 00:00:00 2001 From: Enrico Piovesan Date: Mon, 27 Jul 2026 21:32:56 -0600 Subject: [PATCH 3/3] test: cover datastore lock timeouts --- crates/traverse-runtime/src/data_store.rs | 51 ++++++++++++++++------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/crates/traverse-runtime/src/data_store.rs b/crates/traverse-runtime/src/data_store.rs index ee017e43..6f2e045a 100644 --- a/crates/traverse-runtime/src/data_store.rs +++ b/crates/traverse-runtime/src/data_store.rs @@ -1097,13 +1097,7 @@ mod tests { let root = PathBuf::from(root); let _adapter = LocalFileDataStore::new(&root).expect("child should acquire lock"); fs::write(lock_child_ready_path(&root), "ready").expect("child should signal readiness"); - for _ in 0..500 { - if lock_child_release_path(&root).exists() { - return Ok(()); - } - thread::sleep(Duration::from_millis(10)); - } - Err("parent did not release the lock child".to_string()) + wait_for_lock_child_release(&root, 500) } #[test] @@ -1117,7 +1111,7 @@ mod tests { drop(initial_owner); let mut child = start_lock_child(&root); - wait_for_lock_child(&root).expect("lock child should become ready"); + wait_for_lock_child(&root, 500).expect("lock child should become ready"); let blocked = LocalFileDataStore::new(&root).expect_err("second process must be blocked"); assert_eq!(blocked.code, DataStoreErrorCode::StoreLocked); assert_eq!( @@ -1146,7 +1140,7 @@ mod tests { drop(initial_owner); let mut child = start_lock_child(&root); - wait_for_lock_child(&root).expect("lock child should become ready"); + wait_for_lock_child(&root, 500).expect("lock child should become ready"); child.kill().expect("parent should terminate child"); child.wait().expect("terminated child should exit"); @@ -1261,14 +1255,43 @@ mod tests { .expect("lock child should start") } - fn wait_for_lock_child(root: &Path) -> Result<(), &'static str> { - for _ in 0..500 { - if lock_child_ready_path(root).exists() { - return Ok(()); + #[test] + fn lock_child_waits_report_bounded_timeouts() { + let root = temp_root("lock-child-timeout"); + assert_eq!( + wait_for_lock_child(&root, 0), + Err("lock child did not become ready") + ); + assert_eq!( + wait_for_lock_child_release(&root, 0), + Err("parent did not release the lock child".to_string()) + ); + } + + fn wait_for_lock_child(root: &Path, attempts: usize) -> Result<(), &'static str> { + if wait_for_path(&lock_child_ready_path(root), attempts) { + Ok(()) + } else { + Err("lock child did not become ready") + } + } + + fn wait_for_lock_child_release(root: &Path, attempts: usize) -> Result<(), String> { + if wait_for_path(&lock_child_release_path(root), attempts) { + Ok(()) + } else { + Err("parent did not release the lock child".to_string()) + } + } + + fn wait_for_path(path: &Path, attempts: usize) -> bool { + for _ in 0..attempts { + if path.exists() { + return true; } thread::sleep(Duration::from_millis(10)); } - Err("lock child did not become ready") + false } fn stateful_contract(state_schema: Option) -> CapabilityContract {