From 31d14722a4f3a9184f77df74cfcc6d312a4388a1 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 10:25:10 +0800 Subject: [PATCH 01/40] fix(blaze): persist backend PID handoff This enables explicit destroy and restart cleanup to distinguish a backend that never started from one whose process may still exist. Each spawner creates and syncs an empty PID handoff before Starting ownership is published. During spawn, the child writes and syncs its PID before exec while an inherited file lock prevents cleanup from treating an active handoff as absent. Missing metadata remains an unknown state for compatibility. An unlocked empty handoff proves that no process crossed the spawn boundary. Bubblewrap and Firecracker use the same protocol. Fixes: c33b495450 ("feat(blaze): own sandbox runtime resources") Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/Cargo.lock | 1 + src/blaze/Cargo.toml | 3 + src/blaze/crates/blazed/Cargo.toml | 1 + src/blaze/crates/blazed/src/api.rs | 49 ++- src/blaze/crates/blazed/src/spawner.rs | 344 ++++++++++++++++-- .../crates/blazed/src/spawner/firecracker.rs | 27 +- 6 files changed, 356 insertions(+), 69 deletions(-) diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 36b5e15874..1ed8e6e6d1 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -139,6 +139,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "libc", "serde", "serde_json", "tempfile", diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index c031e1ef38..141359bb35 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -43,6 +43,9 @@ uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } semver = "1.0" +# Platform +libc = "0.2" + # Internal crates blaze-core = { path = "crates/blaze-core" } diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index f4c426ef74..936f81f896 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } +libc = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index c78985cc94..aed3f28d83 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -333,6 +333,36 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result spawner, + None => { + return Err(cleanup_failed_create( + state, + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(format!( + "active backend {} has no registered spawner", + state.active_backend + )), + ) + .await); + } + }; + if let Err(error) = spawner.prepare_spawn(&work_dir).await { + return Err(cleanup_failed_create( + state, + &mut instance, + storage, + None, + false, + error.into(), + ) + .await); + } + instance.backend_ownership = BackendOwnership::Starting; if let Err(error) = instance.persist(&state.state_dir) { instance.backend_ownership = BackendOwnership::NotStarted; @@ -359,25 +389,6 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result spawner, - None => { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(format!( - "active backend {} has no registered spawner", - state.active_backend - )), - ) - .await); - } - }; let spawn = match crate::failpoint::backend("create-spawn") { Ok(()) => { spawner diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index 0e33a3456c..f133d91384 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -25,6 +25,8 @@ use uuid::Uuid; pub use firecracker::FirecrackerSpawner; const TERMINATION_GRACE: Duration = Duration::from_secs(5); +#[cfg(target_os = "linux")] +const PID_HANDOFF_GRACE: Duration = Duration::from_secs(1); const STOPPED_MARKER: &str = "backend.stopped"; /// Result reported when a backend process exits. @@ -135,6 +137,11 @@ impl From for SpawnFailure { /// Factory for owned backend runtime instances. #[async_trait] pub trait BackendSpawner: Send + Sync { + /// Persist backend-specific pre-spawn ownership metadata. + async fn prepare_spawn(&self, _run_dir: &Path) -> Result<()> { + Ok(()) + } + /// Start a new sandbox. async fn spawn( &self, @@ -180,13 +187,20 @@ pub struct BubblewrapSpawner; #[async_trait] impl BackendSpawner for BubblewrapSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("backend.pid")) + } + async fn spawn( &self, request: SpawnRequest, ) -> std::result::Result { tokio::fs::create_dir_all(&request.run_dir).await?; remove_file_if_exists(&request.run_dir.join(STOPPED_MARKER)).await?; - let child = Command::new(&request.binary_path) + let pid_file = request.run_dir.join("backend.pid"); + let mut command = Command::new(&request.binary_path); + command .args([ "--ro-bind", "/", @@ -206,22 +220,12 @@ impl BackendSpawner for BubblewrapSpawner { ]) .env("BLAZE_INSTANCE_ID", request.instance_id.to_string()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - let pid_file = request.run_dir.join("backend.pid"); + .stderr(Stdio::null()); + let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + let child = command.spawn(); + drop(pid_handoff); + let child = child?; let stopped_marker = request.run_dir.join(STOPPED_MARKER); - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { - let owner: DynBackendInstance = Arc::new(ProcessInstance::new( - request.instance_id, - BackendKind::Bubblewrap, - child, - pid_file, - stopped_marker, - )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); - } let instance = ProcessInstance::new( request.instance_id, BackendKind::Bubblewrap, @@ -446,6 +450,147 @@ pub(super) fn stopped_marker(run_dir: &Path) -> PathBuf { run_dir.join(STOPPED_MARKER) } +#[cfg(unix)] +pub(super) struct PidHandoff { + _file: std::fs::File, +} + +#[cfg(not(unix))] +pub(super) struct PidHandoff; + +#[cfg(unix)] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + use std::ffi::CString; + use std::os::fd::FromRawFd; + use std::os::unix::ffi::OsStrExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC | libc::O_NOFOLLOW, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + file.sync_all()?; + if let Some(parent) = pid_file.parent() { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(not(unix))] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + let file = std::fs::File::create(pid_file)?; + file.sync_all()?; + Ok(()) +} + +#[cfg(unix)] +pub(super) fn configure_pid_handoff(command: &mut Command, pid_file: &Path) -> Result { + use std::ffi::CString; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::process::CommandExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CLOEXEC | libc::O_NOFOLLOW, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + let child_fd = file.as_raw_fd(); + // SAFETY: the closure calls only async-signal-safe libc functions and does + // not allocate after fork. The returned guard keeps `child_fd` open and + // locked until `Command::spawn` completes. + unsafe { + command + .as_std_mut() + .pre_exec(move || write_current_pid(child_fd)); + } + Ok(PidHandoff { _file: file }) +} + +#[cfg(not(unix))] +pub(super) fn configure_pid_handoff( + _command: &mut Command, + _pid_file: &Path, +) -> Result { + Ok(PidHandoff) +} + +#[cfg(unix)] +fn write_current_pid(fd: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::lseek(fd, 0, libc::SEEK_SET) } < 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::ftruncate(fd, 0) } != 0 { + return Err(std::io::Error::last_os_error()); + } + write_pid_and_sync(fd) +} + +#[cfg(unix)] +fn write_pid_and_sync(fd: libc::c_int) -> std::io::Result<()> { + let mut buffer = [0_u8; 16]; + let mut cursor = buffer.len(); + cursor -= 1; + buffer[cursor] = b'\n'; + let mut pid = unsafe { libc::getpid() } as u32; + loop { + cursor -= 1; + buffer[cursor] = b'0' + (pid % 10) as u8; + pid /= 10; + if pid == 0 { + break; + } + } + + let mut remaining = &buffer[cursor..]; + while !remaining.is_empty() { + let written = unsafe { + libc::write( + fd, + remaining.as_ptr().cast::(), + remaining.len(), + ) + }; + if written < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if written == 0 { + return Err(std::io::ErrorKind::WriteZero.into()); + } + remaining = &remaining[written as usize..]; + } + if unsafe { libc::fsync(fd) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + async fn cleanup_process_run_dir(instance_id: Uuid, run_dir: &Path, backend: &str) -> Result<()> { let stopped_marker = stopped_marker(run_dir); if stopped_marker.is_file() { @@ -477,17 +622,9 @@ pub(super) async fn terminate_recorded_process( pid_file: &Path, backend: &str, ) -> Result<()> { - let raw = match tokio::fs::read_to_string(pid_file).await { - Ok(raw) => raw, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Err(BlazeError::BackendError { - msg: format!( - "cannot confirm {backend} instance {instance_id} stopped: missing PID metadata {}", - pid_file.display() - ), - }); - } - Err(error) => return Err(error.into()), + let raw = match wait_for_pid_handoff(pid_file).await? { + Some(raw) => raw, + None => return Ok(()), }; let pid: u32 = raw .trim() @@ -537,6 +674,74 @@ pub(super) async fn terminate_recorded_process( Ok(()) } +#[cfg(target_os = "linux")] +async fn wait_for_pid_handoff(pid_file: &Path) -> Result> { + let deadline = Instant::now() + PID_HANDOFF_GRACE; + loop { + match read_pid_handoff(pid_file)? { + PidHandoffState::NotStarted => return Ok(None), + PidHandoffState::Missing => { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: missing PID handoff {}", + pid_file.display() + ), + }); + } + PidHandoffState::Ready(raw) => return Ok(Some(raw)), + PidHandoffState::InProgress => {} + } + if Instant::now() >= deadline { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: PID handoff is still in progress at {}", + pid_file.display() + ), + }); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[cfg(target_os = "linux")] +enum PidHandoffState { + Missing, + NotStarted, + InProgress, + Ready(String), +} + +#[cfg(target_os = "linux")] +fn read_pid_handoff(pid_file: &Path) -> Result { + use std::io::{Read, Seek, SeekFrom}; + use std::os::fd::AsRawFd; + + let mut file = match std::fs::OpenOptions::new().read(true).open(pid_file) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(PidHandoffState::Missing); + } + Err(error) => return Err(error.into()), + }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EAGAIN) + || error.raw_os_error() == Some(libc::EWOULDBLOCK) + { + return Ok(PidHandoffState::InProgress); + } + return Err(error.into()); + } + file.seek(SeekFrom::Start(0))?; + let mut raw = String::new(); + file.read_to_string(&mut raw)?; + if raw.trim().is_empty() { + Ok(PidHandoffState::NotStarted) + } else { + Ok(PidHandoffState::Ready(raw)) + } +} + #[cfg(target_os = "linux")] async fn wait_for_process_exit(process_dir: &Path, timeout: Duration) -> Result { let deadline = Instant::now() + timeout; @@ -720,19 +925,90 @@ mod tests { #[cfg(target_os = "linux")] #[tokio::test] - async fn orphan_cleanup_rejects_missing_pid_without_stop_record() { + async fn orphan_cleanup_accepts_pre_spawn_handoff_without_pid() { + let temp = tempfile::tempdir().expect("temp"); + prepare_pid_handoff(&temp.path().join("backend.pid")).expect("prepare handoff"); + cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .await + .expect("an unlocked empty handoff proves the backend was not started"); + assert!(stopped_marker(temp.path()).is_file()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_rejects_missing_pid_handoff() { let temp = tempfile::tempdir().expect("temp"); let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect_err("missing metadata cannot prove termination"); - assert!(error.to_string().contains("missing PID metadata")); + .expect_err("missing handoff cannot prove backend ownership"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(!stopped_marker(temp.path()).exists()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_an_active_pid_handoff() { + let temp = tempfile::tempdir().expect("temp"); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); - record_backend_stopped(&stopped_marker(temp.path())) + let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect("record stopped"); - cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .expect_err("an active handoff cannot prove the backend absent"); + + assert!( + error + .to_string() + .contains("PID handoff is still in progress") + ); + assert!(!stopped_marker(temp.path()).exists()); + drop(handoff); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn pid_handoff_is_visible_when_spawn_returns() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); + let mut child = command.spawn().expect("spawn child"); + drop(handoff); + wait_for_instance_marker(&child, instance_id).await; + + assert_eq!( + std::fs::read_to_string(&pid_file) + .expect("pid handoff") + .trim(), + child.id().expect("child pid").to_string() + ); + terminate_recorded_process(instance_id, &pid_file, "test") .await - .expect("durable stop record proves termination"); + .expect("terminate handed-off process"); + child.wait().await.expect("reap child"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_pid_handoff_preparation_does_not_start_backend() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("missing").join("backend.pid"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + + assert!(configure_pid_handoff(&mut command, &pid_file).is_err()); + assert!(!pid_file.exists()); } #[cfg(target_os = "linux")] diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index a037ee5c29..efac9e95a5 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -20,7 +20,8 @@ use uuid::Uuid; use super::terminate_recorded_process; use super::{ BackendInstance, BackendSpawner, DynBackendInstance, SpawnFailure, SpawnResult, - record_backend_stopped, remove_file_if_exists, spawn_result, stopped_marker, terminate_child, + configure_pid_handoff, prepare_pid_handoff, record_backend_stopped, remove_file_if_exists, + spawn_result, stopped_marker, terminate_child, }; /// Firecracker backend factory. @@ -54,7 +55,6 @@ impl FirecrackerSpawner { let stopped_marker = stopped_marker(&request.run_dir); remove_if_exists(&api_socket).await?; remove_if_exists(&guest_socket).await?; - remove_if_exists(&pid_file).await?; remove_file_if_exists(&stopped_marker).await?; let fc_config = request .backend @@ -67,23 +67,13 @@ impl FirecrackerSpawner { command.arg("--config-file").arg(config_path); configure_logs(&mut command, &request.run_dir, fc_config.serial_log)?; command.env("BLAZE_INSTANCE_ID", request.instance_id.to_string()); - let mut child = match command.spawn() { + let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + let child = command.spawn(); + drop(pid_handoff); + let mut child = match child { Ok(child) => child, Err(source) => return Err(source.into()), }; - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { - let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( - request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, - )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); - } if let Err(error) = wait_for_socket(&api_socket, &mut child, self.socket_timeout).await { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( request.instance_id, @@ -110,6 +100,11 @@ impl FirecrackerSpawner { #[async_trait] impl BackendSpawner for FirecrackerSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("firecracker.pid")) + } + async fn spawn( &self, request: SpawnRequest, From 328fd10b07106ef73ef3752d9e428c8ad7ac98c8 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 10:25:10 +0800 Subject: [PATCH 02/40] fix(blaze): persist backend PID handoff This enables explicit destroy and restart cleanup to distinguish a backend that never started from one whose process may still exist. Each spawner creates and syncs an empty PID handoff before Starting ownership is published. During spawn, the child writes and syncs its PID before exec while an inherited file lock prevents cleanup from treating an active handoff as absent. Missing metadata remains an unknown state for compatibility. An unlocked empty handoff proves that no process crossed the spawn boundary. Bubblewrap and Firecracker use the same protocol. Fixes: c33b495450 ("feat(blaze): own sandbox runtime resources") Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/Cargo.lock | 1 + src/blaze/Cargo.toml | 3 + src/blaze/crates/blazed/Cargo.toml | 1 + src/blaze/crates/blazed/src/api.rs | 49 ++- src/blaze/crates/blazed/src/spawner.rs | 344 ++++++++++++++++-- .../crates/blazed/src/spawner/firecracker.rs | 27 +- 6 files changed, 356 insertions(+), 69 deletions(-) diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 36b5e15874..1ed8e6e6d1 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -139,6 +139,7 @@ dependencies = [ "http-body-util", "hyper", "hyper-util", + "libc", "serde", "serde_json", "tempfile", diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index c031e1ef38..141359bb35 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -43,6 +43,9 @@ uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } semver = "1.0" +# Platform +libc = "0.2" + # Internal crates blaze-core = { path = "crates/blaze-core" } diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index f4c426ef74..936f81f896 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -34,6 +34,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } +libc = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index c78985cc94..aed3f28d83 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -333,6 +333,36 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result spawner, + None => { + return Err(cleanup_failed_create( + state, + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(format!( + "active backend {} has no registered spawner", + state.active_backend + )), + ) + .await); + } + }; + if let Err(error) = spawner.prepare_spawn(&work_dir).await { + return Err(cleanup_failed_create( + state, + &mut instance, + storage, + None, + false, + error.into(), + ) + .await); + } + instance.backend_ownership = BackendOwnership::Starting; if let Err(error) = instance.persist(&state.state_dir) { instance.backend_ownership = BackendOwnership::NotStarted; @@ -359,25 +389,6 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result spawner, - None => { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(format!( - "active backend {} has no registered spawner", - state.active_backend - )), - ) - .await); - } - }; let spawn = match crate::failpoint::backend("create-spawn") { Ok(()) => { spawner diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index 0e33a3456c..f133d91384 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -25,6 +25,8 @@ use uuid::Uuid; pub use firecracker::FirecrackerSpawner; const TERMINATION_GRACE: Duration = Duration::from_secs(5); +#[cfg(target_os = "linux")] +const PID_HANDOFF_GRACE: Duration = Duration::from_secs(1); const STOPPED_MARKER: &str = "backend.stopped"; /// Result reported when a backend process exits. @@ -135,6 +137,11 @@ impl From for SpawnFailure { /// Factory for owned backend runtime instances. #[async_trait] pub trait BackendSpawner: Send + Sync { + /// Persist backend-specific pre-spawn ownership metadata. + async fn prepare_spawn(&self, _run_dir: &Path) -> Result<()> { + Ok(()) + } + /// Start a new sandbox. async fn spawn( &self, @@ -180,13 +187,20 @@ pub struct BubblewrapSpawner; #[async_trait] impl BackendSpawner for BubblewrapSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("backend.pid")) + } + async fn spawn( &self, request: SpawnRequest, ) -> std::result::Result { tokio::fs::create_dir_all(&request.run_dir).await?; remove_file_if_exists(&request.run_dir.join(STOPPED_MARKER)).await?; - let child = Command::new(&request.binary_path) + let pid_file = request.run_dir.join("backend.pid"); + let mut command = Command::new(&request.binary_path); + command .args([ "--ro-bind", "/", @@ -206,22 +220,12 @@ impl BackendSpawner for BubblewrapSpawner { ]) .env("BLAZE_INSTANCE_ID", request.instance_id.to_string()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn()?; - let pid_file = request.run_dir.join("backend.pid"); + .stderr(Stdio::null()); + let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + let child = command.spawn(); + drop(pid_handoff); + let child = child?; let stopped_marker = request.run_dir.join(STOPPED_MARKER); - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { - let owner: DynBackendInstance = Arc::new(ProcessInstance::new( - request.instance_id, - BackendKind::Bubblewrap, - child, - pid_file, - stopped_marker, - )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); - } let instance = ProcessInstance::new( request.instance_id, BackendKind::Bubblewrap, @@ -446,6 +450,147 @@ pub(super) fn stopped_marker(run_dir: &Path) -> PathBuf { run_dir.join(STOPPED_MARKER) } +#[cfg(unix)] +pub(super) struct PidHandoff { + _file: std::fs::File, +} + +#[cfg(not(unix))] +pub(super) struct PidHandoff; + +#[cfg(unix)] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + use std::ffi::CString; + use std::os::fd::FromRawFd; + use std::os::unix::ffi::OsStrExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CREAT | libc::O_TRUNC | libc::O_CLOEXEC | libc::O_NOFOLLOW, + 0o600, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + file.sync_all()?; + if let Some(parent) = pid_file.parent() { + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +#[cfg(not(unix))] +pub(super) fn prepare_pid_handoff(pid_file: &Path) -> Result<()> { + let file = std::fs::File::create(pid_file)?; + file.sync_all()?; + Ok(()) +} + +#[cfg(unix)] +pub(super) fn configure_pid_handoff(command: &mut Command, pid_file: &Path) -> Result { + use std::ffi::CString; + use std::os::fd::{AsRawFd, FromRawFd}; + use std::os::unix::ffi::OsStrExt; + use std::os::unix::process::CommandExt; + + let pid_path = + CString::new(pid_file.as_os_str().as_bytes()).map_err(|_| BlazeError::BackendError { + msg: format!("PID file path contains a NUL byte: {}", pid_file.display()), + })?; + let fd = unsafe { + libc::open( + pid_path.as_ptr(), + libc::O_RDWR | libc::O_CLOEXEC | libc::O_NOFOLLOW, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().into()); + } + let file = unsafe { std::fs::File::from_raw_fd(fd) }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX) } != 0 { + return Err(std::io::Error::last_os_error().into()); + } + let child_fd = file.as_raw_fd(); + // SAFETY: the closure calls only async-signal-safe libc functions and does + // not allocate after fork. The returned guard keeps `child_fd` open and + // locked until `Command::spawn` completes. + unsafe { + command + .as_std_mut() + .pre_exec(move || write_current_pid(child_fd)); + } + Ok(PidHandoff { _file: file }) +} + +#[cfg(not(unix))] +pub(super) fn configure_pid_handoff( + _command: &mut Command, + _pid_file: &Path, +) -> Result { + Ok(PidHandoff) +} + +#[cfg(unix)] +fn write_current_pid(fd: libc::c_int) -> std::io::Result<()> { + if unsafe { libc::lseek(fd, 0, libc::SEEK_SET) } < 0 { + return Err(std::io::Error::last_os_error()); + } + if unsafe { libc::ftruncate(fd, 0) } != 0 { + return Err(std::io::Error::last_os_error()); + } + write_pid_and_sync(fd) +} + +#[cfg(unix)] +fn write_pid_and_sync(fd: libc::c_int) -> std::io::Result<()> { + let mut buffer = [0_u8; 16]; + let mut cursor = buffer.len(); + cursor -= 1; + buffer[cursor] = b'\n'; + let mut pid = unsafe { libc::getpid() } as u32; + loop { + cursor -= 1; + buffer[cursor] = b'0' + (pid % 10) as u8; + pid /= 10; + if pid == 0 { + break; + } + } + + let mut remaining = &buffer[cursor..]; + while !remaining.is_empty() { + let written = unsafe { + libc::write( + fd, + remaining.as_ptr().cast::(), + remaining.len(), + ) + }; + if written < 0 { + let error = std::io::Error::last_os_error(); + if error.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(error); + } + if written == 0 { + return Err(std::io::ErrorKind::WriteZero.into()); + } + remaining = &remaining[written as usize..]; + } + if unsafe { libc::fsync(fd) } != 0 { + return Err(std::io::Error::last_os_error()); + } + Ok(()) +} + async fn cleanup_process_run_dir(instance_id: Uuid, run_dir: &Path, backend: &str) -> Result<()> { let stopped_marker = stopped_marker(run_dir); if stopped_marker.is_file() { @@ -477,17 +622,9 @@ pub(super) async fn terminate_recorded_process( pid_file: &Path, backend: &str, ) -> Result<()> { - let raw = match tokio::fs::read_to_string(pid_file).await { - Ok(raw) => raw, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Err(BlazeError::BackendError { - msg: format!( - "cannot confirm {backend} instance {instance_id} stopped: missing PID metadata {}", - pid_file.display() - ), - }); - } - Err(error) => return Err(error.into()), + let raw = match wait_for_pid_handoff(pid_file).await? { + Some(raw) => raw, + None => return Ok(()), }; let pid: u32 = raw .trim() @@ -537,6 +674,74 @@ pub(super) async fn terminate_recorded_process( Ok(()) } +#[cfg(target_os = "linux")] +async fn wait_for_pid_handoff(pid_file: &Path) -> Result> { + let deadline = Instant::now() + PID_HANDOFF_GRACE; + loop { + match read_pid_handoff(pid_file)? { + PidHandoffState::NotStarted => return Ok(None), + PidHandoffState::Missing => { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: missing PID handoff {}", + pid_file.display() + ), + }); + } + PidHandoffState::Ready(raw) => return Ok(Some(raw)), + PidHandoffState::InProgress => {} + } + if Instant::now() >= deadline { + return Err(BlazeError::BackendError { + msg: format!( + "cannot confirm backend process ownership: PID handoff is still in progress at {}", + pid_file.display() + ), + }); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[cfg(target_os = "linux")] +enum PidHandoffState { + Missing, + NotStarted, + InProgress, + Ready(String), +} + +#[cfg(target_os = "linux")] +fn read_pid_handoff(pid_file: &Path) -> Result { + use std::io::{Read, Seek, SeekFrom}; + use std::os::fd::AsRawFd; + + let mut file = match std::fs::OpenOptions::new().read(true).open(pid_file) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(PidHandoffState::Missing); + } + Err(error) => return Err(error.into()), + }; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EAGAIN) + || error.raw_os_error() == Some(libc::EWOULDBLOCK) + { + return Ok(PidHandoffState::InProgress); + } + return Err(error.into()); + } + file.seek(SeekFrom::Start(0))?; + let mut raw = String::new(); + file.read_to_string(&mut raw)?; + if raw.trim().is_empty() { + Ok(PidHandoffState::NotStarted) + } else { + Ok(PidHandoffState::Ready(raw)) + } +} + #[cfg(target_os = "linux")] async fn wait_for_process_exit(process_dir: &Path, timeout: Duration) -> Result { let deadline = Instant::now() + timeout; @@ -720,19 +925,90 @@ mod tests { #[cfg(target_os = "linux")] #[tokio::test] - async fn orphan_cleanup_rejects_missing_pid_without_stop_record() { + async fn orphan_cleanup_accepts_pre_spawn_handoff_without_pid() { + let temp = tempfile::tempdir().expect("temp"); + prepare_pid_handoff(&temp.path().join("backend.pid")).expect("prepare handoff"); + cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .await + .expect("an unlocked empty handoff proves the backend was not started"); + assert!(stopped_marker(temp.path()).is_file()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_rejects_missing_pid_handoff() { let temp = tempfile::tempdir().expect("temp"); let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect_err("missing metadata cannot prove termination"); - assert!(error.to_string().contains("missing PID metadata")); + .expect_err("missing handoff cannot prove backend ownership"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(!stopped_marker(temp.path()).exists()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_an_active_pid_handoff() { + let temp = tempfile::tempdir().expect("temp"); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); - record_backend_stopped(&stopped_marker(temp.path())) + let error = cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") .await - .expect("record stopped"); - cleanup_process_run_dir(Uuid::new_v4(), temp.path(), "test") + .expect_err("an active handoff cannot prove the backend absent"); + + assert!( + error + .to_string() + .contains("PID handoff is still in progress") + ); + assert!(!stopped_marker(temp.path()).exists()); + drop(handoff); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn pid_handoff_is_visible_when_spawn_returns() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("backend.pid"); + prepare_pid_handoff(&pid_file).expect("prepare handoff"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + let handoff = configure_pid_handoff(&mut command, &pid_file).expect("configure handoff"); + let mut child = command.spawn().expect("spawn child"); + drop(handoff); + wait_for_instance_marker(&child, instance_id).await; + + assert_eq!( + std::fs::read_to_string(&pid_file) + .expect("pid handoff") + .trim(), + child.id().expect("child pid").to_string() + ); + terminate_recorded_process(instance_id, &pid_file, "test") .await - .expect("durable stop record proves termination"); + .expect("terminate handed-off process"); + child.wait().await.expect("reap child"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_pid_handoff_preparation_does_not_start_backend() { + let temp = tempfile::tempdir().expect("temp"); + let instance_id = Uuid::new_v4(); + let pid_file = temp.path().join("missing").join("backend.pid"); + let mut command = Command::new("sleep"); + command + .arg("60") + .env("BLAZE_INSTANCE_ID", instance_id.to_string()); + + assert!(configure_pid_handoff(&mut command, &pid_file).is_err()); + assert!(!pid_file.exists()); } #[cfg(target_os = "linux")] diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index a037ee5c29..efac9e95a5 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -20,7 +20,8 @@ use uuid::Uuid; use super::terminate_recorded_process; use super::{ BackendInstance, BackendSpawner, DynBackendInstance, SpawnFailure, SpawnResult, - record_backend_stopped, remove_file_if_exists, spawn_result, stopped_marker, terminate_child, + configure_pid_handoff, prepare_pid_handoff, record_backend_stopped, remove_file_if_exists, + spawn_result, stopped_marker, terminate_child, }; /// Firecracker backend factory. @@ -54,7 +55,6 @@ impl FirecrackerSpawner { let stopped_marker = stopped_marker(&request.run_dir); remove_if_exists(&api_socket).await?; remove_if_exists(&guest_socket).await?; - remove_if_exists(&pid_file).await?; remove_file_if_exists(&stopped_marker).await?; let fc_config = request .backend @@ -67,23 +67,13 @@ impl FirecrackerSpawner { command.arg("--config-file").arg(config_path); configure_logs(&mut command, &request.run_dir, fc_config.serial_log)?; command.env("BLAZE_INSTANCE_ID", request.instance_id.to_string()); - let mut child = match command.spawn() { + let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + let child = command.spawn(); + drop(pid_handoff); + let mut child = match child { Ok(child) => child, Err(source) => return Err(source.into()), }; - if let Some(pid) = child.id() - && let Err(error) = tokio::fs::write(&pid_file, format!("{pid}\n")).await - { - let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( - request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, - )); - return Err(SpawnFailure::compensate_started(error.into(), owner).await); - } if let Err(error) = wait_for_socket(&api_socket, &mut child, self.socket_timeout).await { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( request.instance_id, @@ -110,6 +100,11 @@ impl FirecrackerSpawner { #[async_trait] impl BackendSpawner for FirecrackerSpawner { + async fn prepare_spawn(&self, run_dir: &Path) -> Result<()> { + tokio::fs::create_dir_all(run_dir).await?; + prepare_pid_handoff(&run_dir.join("firecracker.pid")) + } + async fn spawn( &self, request: SpawnRequest, From 9586e69e249a6a4c47a9194a12c6399bacc79346 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 11:42:44 +0800 Subject: [PATCH 03/40] fix(blaze): reject incomplete sandbox resets This enables POST /v1/instances/{id}/reset to reject requests until a backend can reset both runtime and storage state. The endpoint now returns 501 without changing lifecycle state, runtime ownership, or pool membership. Warm-claim tests use a test-only pool fixture so candidate validation does not rely on a production reset. A later runtime-reset capability can replace this guard after it resets both backend and storage state. Fixes: 1f0cfac49 Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 3 +- src/blaze/README_zh.md | 2 +- src/blaze/crates/blazed/src/api.rs | 85 ++++++++++++++++++++++++---- src/blaze/crates/blazed/src/error.rs | 4 ++ 4 files changed, 79 insertions(+), 15 deletions(-) diff --git a/src/blaze/README.md b/src/blaze/README.md index 8f4b0c00a1..de3fa07e30 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -104,7 +104,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances` | Create a new sandbox instance | | GET | `/v1/instances/{id}` | Get instance details | | POST | `/v1/instances/{id}/checkpoint` | Checkpoint an instance | -| POST | `/v1/instances/{id}/reset` | Reset instance to checkpoint | +| POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | POST | `/v1/instances/{id}/destroy` | Destroy an instance | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | @@ -148,4 +148,3 @@ src/blaze/ - Linux host with root privileges for sandbox backends ## License - diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index fddef048c4..1af630135f 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -103,7 +103,7 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances` | 创建新 sandbox 实例 | | GET | `/v1/instances/{id}` | 获取实例详情 | | POST | `/v1/instances/{id}/checkpoint` | 对实例做 checkpoint | -| POST | `/v1/instances/{id}/reset` | 将实例重置到 checkpoint | +| POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | POST | `/v1/instances/{id}/destroy` | 销毁实例 | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index aed3f28d83..268705ced8 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -923,6 +923,23 @@ async fn checkpoint(state: &Arc, id: &str) -> Result, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let operation_lock = state.operation_lock(uuid); + let _operation = operation_lock.lock().await; + let map = state + .instances + .lock() + .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; + map.get(&uuid) + .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; + + Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {uuid} cannot be reset until its backend can reset runtime and storage state" + ))) +} + +#[cfg(test)] +async fn return_to_pool_for_test(state: &Arc, id: &str) -> Result<()> { let uuid = parse_uuid(id)?; let operation_lock = state.operation_lock(uuid); let _operation = operation_lock.lock().await; @@ -933,9 +950,6 @@ async fn reset_instance(state: &Arc, id: &str) -> Result, id: &str) -> Result, id: &str) -> Result, id: &str) -> Result>> { @@ -1726,6 +1738,51 @@ mod tests { let _ = std::fs::remove_dir_all(&tmp); } + #[tokio::test] + async fn reset_rejects_state_only_pool_return() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let request = test_request(); + let created = created_json(&state, &request).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + + let error = reset_instance(&state, id) + .await + .expect_err("reset without a runtime implementation must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Running + ); + let key = PoolKey::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + ); + assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 0); + assert!( + state + .backend_instances + .lock() + .expect("owners") + .contains_key(&uuid) + ); + } + #[tokio::test] async fn warm_claim_validates_runtime_and_quarantines_dead_owner() { let temp = tempfile::tempdir().expect("temp"); @@ -1793,7 +1850,9 @@ mod tests { serde_json::from_slice(&cold.into_body().collect().await.expect("body").to_bytes()) .expect("cold json"); let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("return to pool"); + return_to_pool_for_test(&state, &id) + .await + .expect("return to pool"); let warm = create_instance(&state, &request) .await @@ -1804,7 +1863,7 @@ mod tests { assert_eq!(warm["instance"]["id"], id); assert_eq!(warm["start_path"], "warm"); - reset_instance(&state, &id) + return_to_pool_for_test(&state, &id) .await .expect("return live owner"); let owner = state @@ -1893,7 +1952,9 @@ mod tests { assert_eq!(cold["instance"]["backend"], "mock"); assert_eq!(cold["selected_backend"], "mock"); - reset_instance(&state, &id).await.expect("return to pool"); + return_to_pool_for_test(&state, &id) + .await + .expect("return to pool"); let warm = created_json(&state, &request).await; assert_eq!(warm["instance"]["id"], id); assert_eq!(warm["instance"]["backend"], "mock"); @@ -2135,7 +2196,7 @@ mod tests { let request = test_request(); let cold = created_json(&state, &request).await; let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + return_to_pool_for_test(&state, &id).await.expect("warm"); storage.fail_reconstruct.store(true, Ordering::Release); let error = create_instance(&state, &request) @@ -2179,7 +2240,7 @@ mod tests { let request = test_request(); let cold = created_json(&state, &request).await; let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + return_to_pool_for_test(&state, &id).await.expect("warm"); std::fs::remove_file(instances_dir.join(&id).join("mem.bin")).expect("remove artifact"); let replacement = created_json(&state, &request).await; @@ -2416,7 +2477,7 @@ mod tests { let request = test_request(); let cold = created_json(&state, &request).await; let id = cold["instance"]["id"].as_str().expect("id").to_string(); - reset_instance(&state, &id).await.expect("warm"); + return_to_pool_for_test(&state, &id).await.expect("warm"); let pause_hook = crate::failpoint::TestFailpoint::new(&["warm-before-state-commit"]); let create_state = state.clone(); diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index 6c0358eb2b..b2bd1a4f87 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -51,6 +51,9 @@ pub enum BlazeDaemonError { #[error("not found: {0}")] NotFound(String), + #[error("unsupported operation: {0}")] + UnsupportedOperation(String), + #[error("operation requires recovery: {0}")] RecoveryRequired(String), @@ -64,6 +67,7 @@ impl BlazeDaemonError { match self { BlazeDaemonError::BadRequest(_) => 400, BlazeDaemonError::NotFound(_) => 404, + BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::RecoveryRequired(_) => 500, BlazeDaemonError::HttpStatus { status, .. } => *status, BlazeDaemonError::Core(blaze_core::BlazeError::PolicyEvalError { .. }) From b9127d1875070d91bafb25db49e23f41d282596b Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 14:22:27 +0800 Subject: [PATCH 04/40] fix(blaze): reject incomplete checkpoints This prevents POST /v1/instances/{id}/checkpoint from reporting a checkpoint before backend and storage capture are wired. The handler validates instance existence under the operation lock and returns HTTP 501 without changing lifecycle state, rewriting state.json, releasing runtime ownership, or generating a checkpoint ID. The focused test proves the in-memory and persisted state remain unchanged. The checkpoint and restore feature will replace this guard after it can pause the backend, capture runtime and storage data, and compensate partial failures. Fixes: 1f0cfac49 Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 2 +- src/blaze/README_zh.md | 2 +- src/blaze/crates/blazed/src/api.rs | 64 +++++++++++++++++++++++------- 3 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/blaze/README.md b/src/blaze/README.md index de3fa07e30..99aaf5e6e7 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -103,7 +103,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | GET | `/v1/instances` | List all instances | | POST | `/v1/instances` | Create a new sandbox instance | | GET | `/v1/instances/{id}` | Get instance details | -| POST | `/v1/instances/{id}/checkpoint` | Checkpoint an instance | +| POST | `/v1/instances/{id}/checkpoint` | Reserved; returns `501` until backend and storage capture is implemented | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | POST | `/v1/instances/{id}/destroy` | Destroy an instance | | GET | `/v1/pools` | List warm pools | diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index 1af630135f..63403703d9 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -102,7 +102,7 @@ images_dir = "/var/lib/blaze/images" | GET | `/v1/instances` | 列出所有实例 | | POST | `/v1/instances` | 创建新 sandbox 实例 | | GET | `/v1/instances/{id}` | 获取实例详情 | -| POST | `/v1/instances/{id}/checkpoint` | 对实例做 checkpoint | +| POST | `/v1/instances/{id}/checkpoint` | 预留接口；后端和存储快照实现前返回 `501` | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | POST | `/v1/instances/{id}/destroy` | 销毁实例 | | GET | `/v1/pools` | 列出 warm pool | diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 268705ced8..2fd473384c 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -901,25 +901,16 @@ async fn checkpoint(state: &Arc, id: &str) -> Result, id: &str) -> Result>> { @@ -1738,6 +1729,51 @@ mod tests { let _ = std::fs::remove_dir_all(&tmp); } + #[tokio::test] + async fn checkpoint_rejects_state_only_snapshot() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let request = test_request(); + let created = created_json(&state, &request).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let state_path = state.state_dir.join(id).join("state.json"); + let persisted_before = std::fs::read(&state_path).expect("persisted state"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint without backend and storage capture must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Running + ); + assert_eq!( + std::fs::read(state_path).expect("persisted state"), + persisted_before + ); + assert!( + state + .backend_instances + .lock() + .expect("owners") + .contains_key(&uuid) + ); + } + #[tokio::test] async fn reset_rejects_state_only_pool_return() { let temp = tempfile::tempdir().expect("temp"); From a26c7cf702d2cd7408081d0335523da829c9f3ee Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 16:12:45 +0800 Subject: [PATCH 05/40] feat(blaze): manage isolated VM network slots Firecracker sandboxes previously had no daemon-owned network interface path. A policy could start a VM, but it could not request an isolated namespace whose resources followed the backend lifecycle. This adds an opt-in network manager that allocates deterministic namespace, veth, tap, address, forwarding, and translation resources. A host-wide lock serializes allocation across daemon processes, and startup probing checks the required host commands only when a loaded Firecracker policy enables network. The backend persists slot ownership before launch and removes resources after process termination. Restart cleanup uses the PID handoff and stopped marker; missing launch evidence remains retained instead of guessing that deletion is safe. Exit cleanup failures keep the observed result and retry on the next wait. This enables isolated VM network slots through ordinary sandbox creation. It remains disabled by default, and upstream routing and DNS stay operator-owned. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/policy.rs | 13 + src/blaze/crates/blazed/src/daemon.rs | 23 +- src/blaze/crates/blazed/src/spawner.rs | 1 + .../crates/blazed/src/spawner/firecracker.rs | 1214 ++++++++++++++++- src/blaze/crates/blazed/src/spawner/netns.rs | 1079 +++++++++++++++ src/blaze/examples/policies/agent-rl.toml | 1 + src/blaze/examples/policies/agent-tool.toml | 1 + 7 files changed, 2263 insertions(+), 69 deletions(-) create mode 100644 src/blaze/crates/blazed/src/spawner/netns.rs diff --git a/src/blaze/crates/blaze-core/src/policy.rs b/src/blaze/crates/blaze-core/src/policy.rs index bfcbac3aa8..8d1e9acaef 100644 --- a/src/blaze/crates/blaze-core/src/policy.rs +++ b/src/blaze/crates/blaze-core/src/policy.rs @@ -361,6 +361,9 @@ pub struct FirecrackerConfig { /// Enable virtio-vsock (Phase 2+; parsed today but not yet wired). #[serde(default)] pub enable_vsock: bool, + /// Create an isolated network namespace, veth uplink, tap, and NAT. + #[serde(default)] + pub enable_network: bool, /// Capture guest ttyS0 output (Firecracker stdout) to `serial.log`. #[serde(default)] pub serial_log: bool, @@ -377,6 +380,7 @@ impl Default for FirecrackerConfig { Self { boot_args: default_fc_boot_args(), enable_vsock: false, + enable_network: false, serial_log: false, vcpus: None, memory: None, @@ -817,6 +821,7 @@ memory = "4G" [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" +enable_network = true serial_log = true [hooks.on_create] @@ -831,6 +836,13 @@ sequence = ["template-reg:bind-mm-template"] assert_eq!(pf.match_.workload_class, WorkloadClass::AgentRl); assert_eq!(pf.select.backend_priority[0], BackendKind::KataFc); assert!(pf.pool.as_ref().expect("pool").enabled); + assert!( + pf.backend + .firecracker + .as_ref() + .expect("firecracker") + .enable_network + ); } #[test] @@ -929,6 +941,7 @@ memory = "2G" .expect("firecracker config"); assert_eq!(fc.vcpus, Some(2)); assert_eq!(fc.memory, Some("2G".to_string())); + assert!(!fc.enable_network); } #[test] diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index a9616003f3..80fe5e4661 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -37,7 +37,18 @@ pub async fn run(config_path: &Path) -> Result<()> { let pool = PoolManager::new(); let template = TemplateRegistry::new(); let hook = HookRegistry::new(); - let (spawners, active_backend) = build_spawners(&config).await; + let network_required = policy.policies().iter().any(|policy| { + policy + .backend + .firecracker + .as_ref() + .is_some_and(|config| config.enable_network) + && policy + .select + .backend_priority + .contains(&BackendKind::Firecracker) + }); + let (spawners, active_backend) = build_spawners(&config, network_required).await; // Build storage provider if config.storage.provider != "file" && config.storage.provider != "auto" { @@ -123,8 +134,14 @@ fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { /// 1. `firecracker` → [`FirecrackerSpawner`] /// 2. `bubblewrap` → [`BubblewrapSpawner`] /// 3. fallback → [`MockSpawner`] -async fn build_spawners(cfg: &DaemonConfig) -> (SpawnerRegistry, BackendKind) { - let firecracker: DynSpawner = Arc::new(FirecrackerSpawner::new(cfg.storage.images_dir.clone())); +async fn build_spawners( + cfg: &DaemonConfig, + network_required: bool, +) -> (SpawnerRegistry, BackendKind) { + let firecracker: DynSpawner = Arc::new(FirecrackerSpawner::with_network_requirement( + cfg.storage.images_dir.clone(), + network_required, + )); let bubblewrap: DynSpawner = Arc::new(BubblewrapSpawner); let mock: DynSpawner = Arc::new(MockSpawner); let mut spawners = SpawnerRegistry::new(); diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index f133d91384..7dbde624cf 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -2,6 +2,7 @@ //! Backend process ownership and runtime lifecycle abstraction. pub mod firecracker; +mod netns; use std::collections::HashMap; use std::fmt; diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index efac9e95a5..3939ead5b2 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Firecracker process ownership and HTTP API over Unix domain sockets. +use std::io::Write; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::sync::Arc; @@ -16,6 +17,7 @@ use tokio::process::{Child, Command}; use tokio::sync::Mutex; use uuid::Uuid; +use super::netns::{NetworkManager, NetworkSlot}; #[cfg(target_os = "linux")] use super::terminate_recorded_process; use super::{ @@ -24,23 +26,62 @@ use super::{ spawn_result, stopped_marker, terminate_child, }; +const NETWORK_BOOT_IP: &str = "ip=169.254.0.2::169.254.0.1:255.255.255.252::eth0:off"; + /// Firecracker backend factory. pub struct FirecrackerSpawner { images_dir: PathBuf, socket_timeout: Duration, + network: Arc, + network_required: bool, version: Mutex>, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +enum NetworkProcessState { + PreSpawn, + #[default] + Launching, +} + +#[derive(Debug, serde::Deserialize, serde::Serialize)] +struct NetworkRecord { + slot: usize, + owner: Uuid, + #[serde(default)] + process_state: NetworkProcessState, +} + impl FirecrackerSpawner { - /// Create a spawner resolving the guest kernel from `images_dir`. + /// Create a spawner without requiring host networking during startup + /// probing. Individual network-enabled requests still run the full probe. pub fn new(images_dir: PathBuf) -> Self { Self { images_dir, socket_timeout: Duration::from_secs(5), + network: Arc::new(NetworkManager::default()), + network_required: false, version: Mutex::new(None), } } + /// Create a spawner whose startup probe includes network prerequisites + /// when at least one loaded policy enables Firecracker networking. + pub fn with_network_requirement(images_dir: PathBuf, network_required: bool) -> Self { + Self { + network_required, + ..Self::new(images_dir) + } + } + + async fn network_probe_ready(&self) -> Result { + if !self.network_required { + return Ok(true); + } + self.network.probe().await + } + async fn start( &self, request: SpawnRequest, @@ -53,49 +94,262 @@ impl FirecrackerSpawner { let guest_socket = request.run_dir.join("vsock.uds"); let pid_file = request.run_dir.join("firecracker.pid"); let stopped_marker = stopped_marker(&request.run_dir); + let network_file = request.run_dir.join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); remove_if_exists(&api_socket).await?; remove_if_exists(&guest_socket).await?; remove_file_if_exists(&stopped_marker).await?; + remove_if_exists(&network_file).await?; + remove_if_exists(&network_temp_file).await?; let fc_config = request .backend .firecracker .as_ref() .cloned() .unwrap_or_default(); - let mut command = build_launch_command(&request.binary_path, &api_socket); - let config_path = write_vm_config(&self.images_dir, &request, &fc_config, &guest_socket)?; + let network = if fc_config.enable_network { + if !self.network.probe().await? { + return Err(BlazeError::BackendError { + msg: "Firecracker networking is unavailable; it requires Linux root and executable ip, sysctl, and iptables commands".to_string(), + } + .into()); + } + match self + .network + .create(request.instance_id, |slot| { + write_network_metadata(&network_file, slot) + }) + .await + { + Ok(network) => Some(network), + Err(error) => { + let (source, residual) = error.into_parts(); + if let Some(network) = residual { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + Some(network), + self.network.clone(), + )); + return Err(SpawnFailure::compensate_started(source, owner).await); + } + if let Err(cleanup) = remove_if_exists(&network_file).await { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + None, + self.network.clone(), + )); + return Err(SpawnFailure::compensate_started( + BlazeError::BackendError { + msg: format!( + "{source}; network metadata cleanup failed: {cleanup}" + ), + }, + owner, + ) + .await); + } + if let Err(cleanup) = remove_if_exists(&network_temp_file).await { + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + request.instance_id, + None, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + None, + self.network.clone(), + )); + return Err(SpawnFailure::compensate_started( + BlazeError::BackendError { + msg: format!( + "{source}; temporary network metadata cleanup failed: {cleanup}" + ), + }, + owner, + ) + .await); + } + return Err(source.into()); + } + } + } else { + None + }; + + let mut command = build_launch_command(&request.binary_path, network.as_ref(), &api_socket); + let config_path = match write_vm_config( + &self.images_dir, + &request, + &fc_config, + &guest_socket, + network.as_ref(), + ) { + Ok(path) => path, + Err(error) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + error, + ) + .await); + } + }; command.arg("--config-file").arg(config_path); - configure_logs(&mut command, &request.run_dir, fc_config.serial_log)?; + if let Err(error) = configure_logs(&mut command, &request.run_dir, fc_config.serial_log) { + return Err(self + .compensate_before_spawn( + request.instance_id, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + error, + ) + .await); + } command.env("BLAZE_INSTANCE_ID", request.instance_id.to_string()); - let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; + if let Some(slot) = network.as_ref() + && let Err(error) = + write_network_record(&network_file, slot, NetworkProcessState::Launching) + { + return Err(self + .compensate_before_spawn( + request.instance_id, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + error, + ) + .await); + } + let pid_handoff = match configure_pid_handoff(&mut command, &pid_file) { + Ok(pid_handoff) => pid_handoff, + Err(error) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + error, + ) + .await); + } + }; let child = command.spawn(); drop(pid_handoff); let mut child = match child { Ok(child) => child, - Err(source) => return Err(source.into()), + Err(source) => { + return Err(self + .compensate_before_spawn( + request.instance_id, + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + source.into(), + ) + .await); + } }; if let Err(error) = wait_for_socket(&api_socket, &mut child, self.socket_timeout).await { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, + Some(child), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + self.network.clone(), )); return Err(SpawnFailure::compensate_started(error, owner).await); } let instance = FirecrackerInstance::new( request.instance_id, - child, - api_socket, - guest_socket, - pid_file, - stopped_marker, + Some(child), + runtime_files( + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + ), + network, + self.network.clone(), ); Ok(Arc::new(instance)) } + + async fn compensate_before_spawn( + &self, + instance_id: Uuid, + files: FirecrackerRuntimeFiles, + network: Option, + source: BlazeError, + ) -> SpawnFailure { + if network.is_none() { + return SpawnFailure::clean(source); + } + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + instance_id, + None, + files, + network, + self.network.clone(), + )); + SpawnFailure::compensate_started(source, owner).await + } } #[async_trait] @@ -116,6 +370,9 @@ impl BackendSpawner for FirecrackerSpawner { if !binary_path.is_file() || !executable_in_path("unshare") { return Ok(false); } + if !self.network_probe_ready().await? { + return Ok(false); + } match read_backend_version(binary_path).await { Ok(version) => { *self.version.lock().await = Some(version); @@ -129,36 +386,61 @@ impl BackendSpawner for FirecrackerSpawner { } async fn cleanup_orphan(&self, instance_id: Uuid, run_dir: &Path) -> Result<()> { - cleanup_orphan_run_dir(instance_id, run_dir).await + cleanup_orphan_run_dir_with(instance_id, run_dir, &self.network).await } } struct FirecrackerInstance { instance_id: Uuid, child: Mutex>, + exit_result: Mutex>, + files: FirecrackerRuntimeFiles, + network: Mutex>, + network_manager: Arc, + cleanup_complete: AtomicBool, + killed: AtomicBool, +} + +struct FirecrackerRuntimeFiles { api_socket: PathBuf, guest_socket: PathBuf, pid_file: PathBuf, stopped_marker: PathBuf, - killed: AtomicBool, + network_file: PathBuf, +} + +fn runtime_files( + api_socket: PathBuf, + guest_socket: PathBuf, + pid_file: PathBuf, + stopped_marker: PathBuf, + network_file: PathBuf, +) -> FirecrackerRuntimeFiles { + FirecrackerRuntimeFiles { + api_socket, + guest_socket, + pid_file, + stopped_marker, + network_file, + } } impl FirecrackerInstance { fn new( instance_id: Uuid, - child: Child, - api_socket: PathBuf, - guest_socket: PathBuf, - pid_file: PathBuf, - stopped_marker: PathBuf, + child: Option, + files: FirecrackerRuntimeFiles, + network: Option, + network_manager: Arc, ) -> Self { Self { instance_id, - child: Mutex::new(Some(child)), - api_socket, - guest_socket, - pid_file, - stopped_marker, + child: Mutex::new(child), + exit_result: Mutex::new(None), + files, + network: Mutex::new(network), + network_manager, + cleanup_complete: AtomicBool::new(false), killed: AtomicBool::new(false), } } @@ -171,24 +453,29 @@ impl BackendInstance for FirecrackerInstance { } async fn try_wait(&self) -> Result> { - let status = { + let result = { let mut guard = self.child.lock().await; let Some(child) = guard.as_mut() else { - return Ok(Some(SpawnResult { + let result = self.exit_result.lock().await.unwrap_or(SpawnResult { instance_id: self.instance_id, exit_code: None, signal: None, - })); + }); + drop(guard); + self.cleanup().await?; + return Ok(Some(result)); }; let Some(status) = child.try_wait()? else { return Ok(None); }; - record_backend_stopped(&self.stopped_marker).await?; + record_backend_stopped(&self.files.stopped_marker).await?; + let result = spawn_result(self.instance_id, status); + *self.exit_result.lock().await = Some(result); *guard = None; - status + result }; self.cleanup().await?; - Ok(Some(spawn_result(self.instance_id, status))) + Ok(Some(result)) } async fn kill(&self) -> Result<()> { @@ -202,7 +489,7 @@ impl BackendInstance for FirecrackerInstance { if let Some(child) = guard.as_mut() { terminate_child(child, "firecracker").await?; } - record_backend_stopped(&self.stopped_marker).await?; + record_backend_stopped(&self.files.stopped_marker).await?; *guard = None; drop(guard); self.cleanup().await?; @@ -213,9 +500,20 @@ impl BackendInstance for FirecrackerInstance { impl FirecrackerInstance { async fn cleanup(&self) -> Result<()> { - remove_if_exists(&self.api_socket).await?; - remove_if_exists(&self.guest_socket).await?; - remove_if_exists(&self.pid_file).await?; + if self.cleanup_complete.load(Ordering::Acquire) { + return Ok(()); + } + remove_if_exists(&self.files.api_socket).await?; + remove_if_exists(&self.files.guest_socket).await?; + remove_if_exists(&self.files.pid_file).await?; + let mut network = self.network.lock().await; + if let Some(slot) = network.as_ref().cloned() { + self.network_manager.destroy(&slot).await?; + *network = None; + } + remove_if_exists(&self.files.network_file).await?; + remove_if_exists(&network_metadata_temp(&self.files.network_file)).await?; + self.cleanup_complete.store(true, Ordering::Release); Ok(()) } } @@ -225,16 +523,43 @@ fn write_vm_config( request: &SpawnRequest, config: &FirecrackerConfig, guest_socket: &Path, + network: Option<&NetworkSlot>, ) -> Result { let vcpus = config .vcpus .or(request.vm.as_ref().map(|vm| vm.vcpus)) .unwrap_or(1); let memory_mib = resolve_memory(config, request.vm.as_ref())?; + let mut boot_args = config.boot_args.clone(); + if network.is_some() { + let network_arguments = boot_args + .split_whitespace() + .filter(|argument| argument.starts_with("ip=")) + .collect::>(); + match network_arguments.as_slice() { + [] => { + boot_args.push(' '); + boot_args.push_str(NETWORK_BOOT_IP); + } + [argument] if *argument == NETWORK_BOOT_IP => {} + arguments => { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker networking requires exactly {NETWORK_BOOT_IP:?}, found {}", + arguments + .iter() + .map(|argument| format!("{argument:?}")) + .collect::>() + .join(", ") + ), + }); + } + } + } let mut value = serde_json::json!({ "boot-source": { "kernel_image_path": path_string(&images_dir.join("vmlinux"), "vmlinux")?, - "boot_args": config.boot_args + "boot_args": boot_args }, "drives": [{ "drive_id": "rootfs", @@ -253,6 +578,13 @@ fn write_vm_config( "uds_path": path_string(guest_socket, "guest socket")? }); } + if let Some(network) = network { + value["network-interfaces"] = serde_json::json!([{ + "iface_id": "eth0", + "guest_mac": "02:FC:00:00:00:02", + "host_dev_name": network.tap_name() + }]); + } let path = request.run_dir.join("vmconfig.json"); std::fs::write( &path, @@ -276,9 +608,26 @@ fn resolve_memory(config: &FirecrackerConfig, vm: Option<&VmConfig>) -> Result Command { +fn build_launch_command( + binary: &Path, + network: Option<&NetworkSlot>, + api_socket: &Path, +) -> Command { #[cfg(target_os = "linux")] - let mut command = { + let mut command = if let Some(network) = network { + let mut command = Command::new("ip"); + command + .arg("netns") + .arg("exec") + .arg(network.netns()) + .arg("unshare") + .arg("--mount") + .arg("--propagation") + .arg("private") + .arg("--") + .arg(binary); + command + } else { let mut command = Command::new("unshare"); command .arg("--mount") @@ -289,7 +638,10 @@ fn build_launch_command(binary: &Path, api_socket: &Path) -> Command { command }; #[cfg(not(target_os = "linux"))] - let mut command = Command::new(binary); + let mut command = { + let _ = network; + Command::new(binary) + }; command.arg("--api-sock").arg(api_socket); command.arg("--id").arg(format!( "fc-{}", @@ -400,6 +752,57 @@ async fn remove_if_exists(path: &Path) -> Result<()> { } } +fn write_network_metadata(path: &Path, network: &NetworkSlot) -> Result<()> { + write_network_record(path, network, NetworkProcessState::PreSpawn) +} + +fn write_network_record( + path: &Path, + network: &NetworkSlot, + process_state: NetworkProcessState, +) -> Result<()> { + let parent = path.parent().ok_or_else(|| BlazeError::BackendError { + msg: format!("network metadata has no parent: {}", path.display()), + })?; + let temporary = network_metadata_temp(path); + (|| -> Result<()> { + let bytes = serde_json::to_vec_pretty(&NetworkRecord { + slot: network.slot(), + owner: network.owner(), + process_state, + }) + .map_err(|error| BlazeError::BackendError { + msg: format!("serialize network metadata: {error}"), + })?; + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&temporary)?; + file.write_all(&bytes)?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all()?; + Ok(()) + })() +} + +fn read_network_metadata(path: &Path) -> Result<(NetworkSlot, NetworkProcessState)> { + let record: NetworkRecord = serde_json::from_slice(&std::fs::read(path)?).map_err(|error| { + BlazeError::BackendError { + msg: format!("parse network metadata {}: {error}", path.display()), + } + })?; + Ok(( + NetworkSlot::from_record(record.slot, record.owner)?, + record.process_state, + )) +} + +fn network_metadata_temp(path: &Path) -> PathBuf { + path.with_extension("json.tmp") +} + fn validate_regular_file(path: &Path, label: &str) -> Result<()> { if !path.is_file() { return Err(BlazeError::BackendError { @@ -457,30 +860,81 @@ fn path_string<'a>(path: &'a Path, label: &str) -> Result<&'a str> { }) } -pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> Result<()> { +async fn cleanup_orphan_run_dir_with( + instance_id: Uuid, + run_dir: &Path, + network_manager: &NetworkManager, +) -> Result<()> { let stopped_marker = stopped_marker(run_dir); - if stopped_marker.is_file() { - return Ok(()); - } let pid_file = run_dir.join("firecracker.pid"); - #[cfg(target_os = "linux")] - { - terminate_recorded_process(instance_id, &pid_file, "firecracker").await?; - } - #[cfg(not(target_os = "linux"))] - { - let _ = instance_id; - if pid_file.exists() { - return Err(BlazeError::BackendError { - msg: format!( - "cannot validate Firecracker orphan {} outside Linux", - pid_file.display() - ), - }); + let network_file = run_dir.join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + let record_path = if network_file.is_file() { + Some(network_file.as_path()) + } else if network_temp_file.is_file() { + Some(network_temp_file.as_path()) + } else { + None + }; + let network_record = match record_path { + Some(path) => match read_network_metadata(path) { + Ok((network, state)) => { + if network.owner() != instance_id { + return Err(BlazeError::BackendError { + msg: format!( + "network record owner {} does not match instance {instance_id}", + network.owner() + ), + }); + } + Some((network, Some(state))) + } + Err(error) if path == network_temp_file.as_path() && !network_file.exists() => { + match network_manager.find_by_owner(instance_id).await? { + // The namespace name proves ownership, but it cannot prove + // whether the backend crossed the spawn boundary. + Some(network) => Some((network, None)), + None => return Err(error), + } + } + Err(error) => return Err(error), + }, + None => network_manager + .find_by_owner(instance_id) + .await? + .map(|network| (network, None)), + }; + let process_may_exist = pid_file.exists() + || network_record + .as_ref() + .is_none_or(|(_, state)| *state != Some(NetworkProcessState::PreSpawn)); + if !stopped_marker.is_file() { + #[cfg(target_os = "linux")] + { + if process_may_exist { + terminate_recorded_process(instance_id, &pid_file, "firecracker").await?; + } + } + #[cfg(not(target_os = "linux"))] + { + let _ = instance_id; + if process_may_exist { + return Err(BlazeError::BackendError { + msg: format!( + "cannot validate Firecracker orphan {} outside Linux", + pid_file.display() + ), + }); + } } + record_backend_stopped(&stopped_marker).await?; } - record_backend_stopped(&stopped_marker).await?; + if let Some((network, _)) = network_record { + network_manager.destroy(&network).await?; + remove_if_exists(&network_file).await?; + } + remove_if_exists(&network_temp_file).await?; remove_if_exists(&run_dir.join("api.sock")).await?; remove_if_exists(&run_dir.join("vsock.uds")).await?; remove_if_exists(&pid_file).await?; @@ -489,8 +943,12 @@ pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> #[cfg(test)] mod tests { + use std::collections::VecDeque; + use blaze_core::storage::StorageSlot; + use crate::spawner::netns::{IpCommandRunner, IpOutput, NetworkManager, test_network_slot}; + use super::*; #[test] @@ -520,6 +978,7 @@ mod tests { &request, &FirecrackerConfig::default(), &temp.path().join("guest.sock"), + None, ) .expect("write config"); let value: serde_json::Value = @@ -528,6 +987,531 @@ mod tests { assert!(value.get("network-interfaces").is_none()); } + #[test] + fn vm_config_wires_an_allocated_network_slot() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + + let path = write_vm_config( + &temp.path().join("images"), + &request, + &FirecrackerConfig::default(), + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect("write config"); + let value: serde_json::Value = + serde_json::from_slice(&std::fs::read(path).expect("read config")) + .expect("parse config"); + assert_eq!(value["network-interfaces"][0]["iface_id"], "eth0"); + assert_eq!(value["network-interfaces"][0]["host_dev_name"], "tap0"); + assert!( + value["boot-source"]["boot_args"] + .as_str() + .expect("boot args") + .contains("::eth0:off") + ); + } + + #[test] + fn vm_config_accepts_the_matching_network_boot_argument() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: format!("console=ttyS0 {NETWORK_BOOT_IP}"), + ..FirecrackerConfig::default() + }; + + write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect("matching network boot argument"); + } + + #[test] + fn vm_config_rejects_an_incompatible_network_boot_argument() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: "console=ttyS0 ip=dhcp".to_string(), + ..FirecrackerConfig::default() + }; + + let error = write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect_err("incompatible network boot argument"); + + assert!(error.to_string().contains("requires")); + assert!(error.to_string().contains("ip=dhcp")); + } + + #[test] + fn vm_config_rejects_conflicting_network_boot_arguments() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let network = test_network_slot(0); + let config = FirecrackerConfig { + boot_args: format!("console=ttyS0 {NETWORK_BOOT_IP} ip=dhcp"), + ..FirecrackerConfig::default() + }; + + let error = write_vm_config( + &temp.path().join("images"), + &request, + &config, + &temp.path().join("guest.sock"), + Some(&network), + ) + .expect_err("conflicting network boot arguments"); + + assert!(error.to_string().contains("exactly")); + assert!(error.to_string().contains("ip=dhcp")); + } + + #[test] + fn network_metadata_is_published_atomically() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + let slot = test_network_slot(7); + + write_network_metadata(&path, &slot).expect("write metadata"); + + let (stored, state) = read_network_metadata(&path).expect("parse metadata"); + assert_eq!(stored, slot); + assert_eq!(state, NetworkProcessState::PreSpawn); + assert!(!network_metadata_temp(&path).exists()); + } + + #[test] + fn network_metadata_records_launch_intent_before_spawn() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + let slot = test_network_slot(7); + write_network_metadata(&path, &slot).expect("write pre-spawn metadata"); + + write_network_record(&path, &slot, NetworkProcessState::Launching) + .expect("record launch intent"); + + let (stored, state) = read_network_metadata(&path).expect("parse metadata"); + assert_eq!(stored, slot); + assert_eq!(state, NetworkProcessState::Launching); + assert!(!network_metadata_temp(&path).exists()); + } + + #[test] + fn network_metadata_rejects_out_of_range_slots() { + let temp = tempfile::tempdir().expect("temp"); + let path = temp.path().join("network.json"); + std::fs::write( + &path, + br#"{"slot":16383,"owner":"00000000-0000-0000-0000-000000000001"}"#, + ) + .expect("metadata"); + + let error = read_network_metadata(&path).expect_err("invalid slot"); + + assert!(error.to_string().contains("outside")); + } + + #[tokio::test] + async fn network_cleanup_failure_retains_a_retryable_backend_owner() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let slot = test_network_slot(0); + write_network_metadata(&network_file, &slot).expect("network metadata"); + let namespace = format!("{}\n", slot.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_failure("delete peer failed"), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = Arc::new(NetworkManager::with_runner(runner.clone())); + let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( + slot.owner(), + None, + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + network_file.clone(), + ), + Some(slot.clone()), + network_manager, + )); + + owner.kill().await.expect_err("first cleanup must fail"); + assert!(network_file.exists()); + owner.kill().await.expect("retry cleanup"); + assert!(!network_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", slot.netns()]) + ); + assert!( + !runner + .calls() + .iter() + .any(|args| args == &["link", "del", "blz-veth-0"]) + ); + } + + #[tokio::test] + async fn try_wait_retries_cleanup_after_observing_process_exit() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let slot = test_network_slot(0); + write_network_metadata(&network_file, &slot).expect("network metadata"); + let namespace = format!("{}\n", slot.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_failure("delete peer failed"), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let child = Command::new("sh") + .arg("-c") + .arg("exit 7") + .spawn() + .expect("spawn child"); + let instance = FirecrackerInstance::new( + slot.owner(), + Some(child), + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + network_file.clone(), + ), + Some(slot.clone()), + Arc::new(NetworkManager::with_runner(runner.clone())), + ); + + let first_error = loop { + match instance.try_wait().await { + Ok(None) => tokio::time::sleep(Duration::from_millis(5)).await, + Ok(Some(result)) => { + panic!("cleanup failure must not report completion: {result:?}") + } + Err(error) => break error, + } + }; + assert!(first_error.to_string().contains("delete peer failed")); + assert!(network_file.exists()); + + let result = instance + .try_wait() + .await + .expect("retry cleanup") + .expect("completed process"); + assert_eq!(result.exit_code, Some(7)); + assert!(!network_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", slot.netns()]) + ); + } + + #[tokio::test] + async fn stopped_orphan_still_releases_recorded_network() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("orphan cleanup"); + + assert!(!network_file.exists()); + let calls = runner.calls(); + assert!(calls.iter().any(|args| { + args == &[ + "netns", + "exec", + network.netns(), + "ip", + "link", + "del", + "blz-vpeer-0", + ] + })); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[tokio::test] + async fn orphan_cleanup_recovers_a_complete_temporary_network_record() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + let network = test_network_slot(0); + let bytes = serde_json::to_vec(&NetworkRecord { + slot: network.slot(), + owner: network.owner(), + process_state: NetworkProcessState::PreSpawn, + }) + .expect("serialize metadata"); + std::fs::write(&network_temp_file, bytes).expect("temporary metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("orphan cleanup"); + + assert!(!network_file.exists()); + assert!(!network_temp_file.exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_a_truncated_network_record_without_pid_proof() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network_temp_file = network_metadata_temp(&network_file); + std::fs::write(&network_temp_file, b"{").expect("truncated metadata"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("unknown launch state must fail closed"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(network_temp_file.exists()); + assert!(!stopped_marker(temp.path()).exists()); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn orphan_cleanup_retains_an_unrecorded_namespace_without_pid_proof() { + let temp = tempfile::tempdir().expect("temp"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("unknown launch state must fail closed"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(!stopped_marker(temp.path()).exists()); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + + #[tokio::test] + async fn stopped_orphan_releases_an_unrecorded_owner_namespace() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("stopped process permits network recovery"); + + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[tokio::test] + async fn network_record_owner_mismatch_issues_no_host_commands() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let runner = Arc::new(TestIpRunner::default()); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(Uuid::from_u128(2), temp.path(), &network_manager) + .await + .expect_err("mismatched owner must fail"); + + assert!(error.to_string().contains("does not match instance")); + assert!(network_file.exists()); + assert!(runner.calls().is_empty()); + } + + #[tokio::test] + async fn stale_network_record_does_not_delete_a_reused_slot() { + let temp = tempfile::tempdir().expect("temp"); + record_backend_stopped(&stopped_marker(temp.path())) + .await + .expect("stopped marker"); + let network_file = temp.path().join("network.json"); + let old_network = test_network_slot(0); + write_network_metadata(&network_file, &old_network).expect("network metadata"); + let new_network = + NetworkSlot::from_record(0, Uuid::from_u128(2)).expect("new network owner"); + let namespace = format!("{}\n", new_network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ip_success( + namespace.as_bytes(), + )])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(old_network.owner(), temp.path(), &network_manager) + .await + .expect("retire stale record"); + + assert!(!network_file.exists()); + let calls = runner.calls(); + assert_eq!(calls, vec![vec!["netns".to_string(), "list".to_string()]]); + } + + #[tokio::test] + async fn pre_spawn_orphan_releases_network_without_pid_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_metadata(&network_file, &network).expect("network metadata"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("pre-spawn cleanup"); + + assert!(!network_file.exists()); + assert!(stopped_marker(temp.path()).exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn unconfirmed_process_ownership_retains_network_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_record(&network_file, &network, NetworkProcessState::Launching) + .expect("network metadata"); + let runner = Arc::new(TestIpRunner::default()); + let network_manager = NetworkManager::with_runner(runner.clone()); + + let error = cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect_err("missing process metadata must block cleanup"); + + assert!(error.to_string().contains("missing PID handoff")); + assert!(network_file.exists()); + assert!(runner.calls().is_empty()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn launch_intent_with_empty_handoff_releases_network() { + let temp = tempfile::tempdir().expect("temp"); + let network_file = temp.path().join("network.json"); + let network = test_network_slot(0); + write_network_record(&network_file, &network, NetworkProcessState::Launching) + .expect("network metadata"); + prepare_pid_handoff(&temp.path().join("firecracker.pid")).expect("prepare PID handoff"); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(TestIpRunner::with_responses([ + ip_success(namespace.as_bytes()), + ip_success(b""), + ip_success(b""), + ])); + let network_manager = NetworkManager::with_runner(runner.clone()); + + cleanup_orphan_run_dir_with(network.owner(), temp.path(), &network_manager) + .await + .expect("empty handoff proves launch did not start"); + + assert!(!network_file.exists()); + assert!(stopped_marker(temp.path()).exists()); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "del", network.netns()]) + ); + } + #[test] fn serial_log_rotates_before_reuse() { let temp = tempfile::tempdir().expect("temp"); @@ -562,6 +1546,27 @@ mod tests { assert!(is_executable_file(&tool)); } + #[tokio::test] + async fn backend_probe_skips_network_checks_when_no_policy_enables_them() { + let temp = tempfile::tempdir().expect("temp"); + let called = Arc::new(AtomicBool::new(false)); + let network = Arc::new(NetworkManager::with_runner(Arc::new( + UnavailableNetworkRunner { + called: called.clone(), + }, + ))); + let spawner = FirecrackerSpawner { + images_dir: temp.path().join("images"), + socket_timeout: Duration::from_secs(1), + network, + network_required: false, + version: Mutex::new(None), + }; + + assert!(spawner.network_probe_ready().await.expect("probe")); + assert!(!called.load(Ordering::Acquire)); + } + #[tokio::test] async fn start_failure_terminates_child_and_removes_process_metadata() { let temp = tempfile::tempdir().expect("temp"); @@ -578,11 +1583,16 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( Uuid::new_v4(), - child, - temp.path().join("api.sock"), - temp.path().join("guest.sock"), - pid_file.clone(), - stopped_marker(temp.path()), + Some(child), + runtime_files( + temp.path().join("api.sock"), + temp.path().join("guest.sock"), + pid_file.clone(), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + None, + Arc::new(NetworkManager::default()), )); let failure = SpawnFailure::compensate_started( BlazeError::BackendError { @@ -626,4 +1636,76 @@ mod tests { vm: None, } } + + #[derive(Default)] + struct TestIpRunner { + responses: std::sync::Mutex>, + calls: std::sync::Mutex>>, + } + + struct UnavailableNetworkRunner { + called: Arc, + } + + #[async_trait] + impl IpCommandRunner for UnavailableNetworkRunner { + async fn output(&self, _args: &[String], _timeout: Duration) -> Result { + self.called.store(true, Ordering::Release); + Ok(ip_failure("network commands unavailable")) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, _name: &str) -> bool { + false + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + false + } + } + + impl TestIpRunner { + fn with_responses(responses: [IpOutput; N]) -> Self { + Self { + responses: std::sync::Mutex::new(responses.into()), + calls: std::sync::Mutex::new(Vec::new()), + } + } + + fn calls(&self) -> Vec> { + self.calls.lock().expect("calls lock").clone() + } + } + + #[async_trait] + impl IpCommandRunner for TestIpRunner { + async fn output(&self, args: &[String], _timeout: Duration) -> Result { + self.calls.lock().expect("calls lock").push(args.to_vec()); + Ok(self + .responses + .lock() + .expect("responses lock") + .pop_front() + .unwrap_or_else(|| ip_success(b""))) + } + } + + fn ip_success(stdout: &[u8]) -> IpOutput { + IpOutput { + success: true, + status: "exit status: 0".to_string(), + stdout: stdout.to_vec(), + stderr: Vec::new(), + } + } + + fn ip_failure(stderr: &str) -> IpOutput { + IpOutput { + success: false, + status: "exit status: 1".to_string(), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } + } } diff --git a/src/blaze/crates/blazed/src/spawner/netns.rs b/src/blaze/crates/blazed/src/spawner/netns.rs new file mode 100644 index 0000000000..24441a51fd --- /dev/null +++ b/src/blaze/crates/blazed/src/spawner/netns.rs @@ -0,0 +1,1079 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Per-VM network namespace allocation and compensated setup. + +use std::collections::HashSet; +use std::fmt; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use async_trait::async_trait; +use blaze_core::{BlazeError, Result}; +use thiserror::Error; +use tokio::process::Command; +use uuid::Uuid; + +const NET_VETH_BASE: usize = 4; +const NET_VETH_TOP: usize = 0x1_0000; +const NET_MAX_SLOT: usize = (NET_VETH_TOP - NET_VETH_BASE) / 4; +const HOST_NETWORK_LOCK: &str = "/run/lock/blaze-network.lock"; +const HOST_LOCK_RETRY: Duration = Duration::from_millis(10); + +#[derive(Debug, Default)] +struct SlotState { + used: HashSet, + next: usize, +} + +#[derive(Debug, Clone)] +pub(super) struct IpOutput { + pub(super) success: bool, + pub(super) status: String, + pub(super) stdout: Vec, + pub(super) stderr: Vec, +} + +#[async_trait] +pub(super) trait IpCommandRunner: Send + Sync { + async fn output(&self, args: &[String], timeout: Duration) -> Result; + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, _name: &str) -> bool { + true + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + true + } +} + +struct SystemIpCommandRunner; + +#[async_trait] +impl IpCommandRunner for SystemIpCommandRunner { + async fn output(&self, args: &[String], timeout: Duration) -> Result { + let mut command = Command::new("ip"); + command.kill_on_drop(true).env("LC_ALL", "C").args(args); + let output = tokio::time::timeout(timeout, command.output()) + .await + .map_err(|_| BlazeError::BackendError { + msg: format!("ip {} timed out", args.join(" ")), + })??; + Ok(IpOutput { + success: output.status.success(), + status: output.status.to_string(), + stdout: output.stdout, + stderr: output.stderr, + }) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, name: &str) -> bool { + executable_in_path(name) + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + // The current implementation relies on root-owned netns mounts, + // tap creation, forwarding changes, and NAT rules. + unsafe { libc::geteuid() == 0 } + } +} + +/// Process-local allocator and lifecycle owner for Blaze network namespaces. +pub(super) struct NetworkManager { + state: Mutex, + command_timeout: Duration, + runner: Arc, + coordination_file: Option, +} + +impl fmt::Debug for NetworkManager { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("NetworkManager") + .field("command_timeout", &self.command_timeout) + .finish_non_exhaustive() + } +} + +impl Default for NetworkManager { + fn default() -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(5), + runner: Arc::new(SystemIpCommandRunner), + coordination_file: Some(PathBuf::from(HOST_NETWORK_LOCK)), + } + } +} + +struct HostNetworkGuard { + #[cfg(unix)] + _file: Option, +} + +/// One fully configured per-VM network namespace. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct NetworkSlot { + slot: usize, + owner: Uuid, + netns: String, + tap_name: String, + veth_host: String, + veth_peer: String, +} + +/// Network setup failure with an optional residual slot owner. +#[derive(Debug, Error)] +#[error("{source}")] +pub(super) struct NetworkCreateError { + #[source] + source: BlazeError, + residual: Option, +} + +impl NetworkCreateError { + fn clean(source: BlazeError) -> Self { + Self { + source, + residual: None, + } + } + + fn with_residual(source: BlazeError, residual: NetworkSlot) -> Self { + Self { + source, + residual: Some(residual), + } + } + + /// Split the setup error from any network slot that still needs cleanup. + pub(super) fn into_parts(self) -> (BlazeError, Option) { + (self.source, self.residual) + } +} + +impl From for NetworkCreateError { + fn from(source: BlazeError) -> Self { + Self::clean(source) + } +} + +impl NetworkSlot { + pub(super) fn from_record(slot: usize, owner: Uuid) -> Result { + if slot >= NET_MAX_SLOT { + return Err(BlazeError::BackendError { + msg: format!("network slot {slot} is outside 0..{NET_MAX_SLOT}"), + }); + } + Ok(network_slot(slot, owner)) + } + + pub(super) fn slot(&self) -> usize { + self.slot + } + + pub(super) fn owner(&self) -> Uuid { + self.owner + } + + /// Network namespace name. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] + pub(super) fn netns(&self) -> &str { + &self.netns + } + + /// Tap device visible inside the namespace. + pub(super) fn tap_name(&self) -> &str { + &self.tap_name + } +} + +impl NetworkManager { + #[cfg(test)] + pub(super) fn with_runner(runner: Arc) -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(1), + runner, + coordination_file: None, + } + } + + #[cfg(test)] + pub(super) fn with_runner_and_lock( + runner: Arc, + coordination_file: PathBuf, + ) -> Self { + Self { + state: Mutex::new(SlotState::default()), + command_timeout: Duration::from_secs(1), + runner, + coordination_file: Some(coordination_file), + } + } + + /// Check the commands and host conditions required by the network path. + pub(super) async fn probe(&self) -> Result { + #[cfg(not(target_os = "linux"))] + { + Ok(false) + } + #[cfg(target_os = "linux")] + { + if !self.runner.has_network_admin() + || ["ip", "sysctl", "iptables"] + .iter() + .any(|command| !self.runner.executable_in_path(command)) + { + return Ok(false); + } + let args = vec!["netns".to_string(), "list".to_string()]; + Ok(self.run_ip_output(&args).await?.success) + } + } + + /// Create an isolated namespace, veth uplink, tap, route, and NAT rule. + /// + /// `record` runs immediately after this manager creates the namespace and + /// before any dependent resources are added. Callers use it to publish + /// ownership without ever recording a namespace created by another process. + pub(super) async fn create( + &self, + owner: Uuid, + mut record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + let _host_guard = self.acquire_host_guard().await?; + let mut blocked = self.existing_slots().await?; + let (slot, network) = loop { + let slot = self.allocate(&blocked)?; + let network = network_slot(slot, owner); + let add_namespace = vec!["netns".into(), "add".into(), network.netns.clone()]; + match self.run_ip(&add_namespace).await { + Ok(()) => { + if let Err(error) = record(&network) { + return self.fail_setup(&network, error).await; + } + break (slot, network); + } + Err(error) => { + self.release(slot); + let refreshed = self.existing_slots().await?; + if refreshed.contains(&slot) { + blocked.extend(refreshed); + continue; + } + return Err(NetworkCreateError::clean(error)); + } + } + }; + let (host_ip, peer_ip) = veth_ips(slot); + let add_veth = vec![ + "link".into(), + "add".into(), + network.veth_host.clone(), + "type".into(), + "veth".into(), + "peer".into(), + "name".into(), + network.veth_peer.clone(), + "netns".into(), + network.netns.clone(), + ]; + if let Err(error) = self.run_ip(&add_veth).await { + return self.fail_setup(&network, error).await; + } + let host_steps = vec![ + vec![ + "addr".into(), + "add".into(), + format!("{host_ip}/30"), + "dev".into(), + network.veth_host.clone(), + ], + vec![ + "link".into(), + "set".into(), + network.veth_host.clone(), + "up".into(), + ], + ]; + for args in host_steps { + if let Err(error) = self.run_ip(&args).await { + return self.fail_setup(&network, error).await; + } + } + + let ns_steps = vec![ + vec![ + "ip".into(), + "addr".into(), + "add".into(), + format!("{peer_ip}/30"), + "dev".into(), + network.veth_peer.clone(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + network.veth_peer.clone(), + "up".into(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + "lo".into(), + "up".into(), + ], + vec![ + "ip".into(), + "tuntap".into(), + "add".into(), + network.tap_name.clone(), + "mode".into(), + "tap".into(), + ], + vec![ + "ip".into(), + "addr".into(), + "add".into(), + "169.254.0.1/30".into(), + "dev".into(), + network.tap_name.clone(), + ], + vec![ + "ip".into(), + "link".into(), + "set".into(), + network.tap_name.clone(), + "up".into(), + ], + vec![ + "ip".into(), + "route".into(), + "add".into(), + "default".into(), + "via".into(), + host_ip, + ], + vec!["sysctl".into(), "-w".into(), "net.ipv4.ip_forward=1".into()], + vec![ + "iptables".into(), + "-t".into(), + "nat".into(), + "-A".into(), + "POSTROUTING".into(), + "-s".into(), + "169.254.0.2".into(), + "-o".into(), + network.veth_peer.clone(), + "-j".into(), + "SNAT".into(), + "--to".into(), + peer_ip, + ], + ]; + for command in ns_steps { + if let Err(error) = self.run_in_namespace(&network.netns, &command).await { + return self.fail_setup(&network, error).await; + } + } + Ok(network) + } + + /// Remove all resources for a slot and return it to the allocator. + pub(super) async fn destroy(&self, network: &NetworkSlot) -> Result<()> { + let _host_guard = self.acquire_host_guard().await?; + self.cleanup_commands(network).await?; + self.release(network.slot); + Ok(()) + } + + async fn acquire_host_guard(&self) -> Result { + let Some(path) = self.coordination_file.as_deref() else { + return Ok(HostNetworkGuard { + #[cfg(unix)] + _file: None, + }); + }; + acquire_host_guard(path, self.command_timeout).await + } + + fn allocate(&self, blocked: &HashSet) -> Result { + let mut state = self.state.lock().map_err(|_| BlazeError::BackendError { + msg: "network slot allocator lock poisoned".to_string(), + })?; + for offset in 0..NET_MAX_SLOT { + let slot = (state.next + offset) % NET_MAX_SLOT; + if !blocked.contains(&slot) && state.used.insert(slot) { + state.next = (slot + 1) % NET_MAX_SLOT; + return Ok(slot); + } + } + Err(BlazeError::BackendError { + msg: format!("network slots exhausted (max {NET_MAX_SLOT})"), + }) + } + + async fn existing_slots(&self) -> Result> { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(parse_existing_slots(&output.stdout)) + } + + fn release(&self, slot: usize) { + if let Ok(mut state) = self.state.lock() { + state.used.remove(&slot); + } + } + + async fn cleanup_commands(&self, network: &NetworkSlot) -> Result<()> { + if !self.namespace_exists(&network.netns).await? { + return Ok(()); + } + self.run_in_namespace_cleanup( + &network.netns, + &[ + "ip".into(), + "link".into(), + "del".into(), + network.veth_peer.clone(), + ], + ) + .await?; + self.delete_namespace(&network.netns).await?; + Ok(()) + } + + pub(super) async fn find_by_owner(&self, owner: Uuid) -> Result> { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(parse_existing_networks(&output.stdout) + .into_iter() + .find(|network| network.owner == owner)) + } + + async fn namespace_exists(&self, name: &str) -> Result { + let args = vec!["netns".to_string(), "list".to_string()]; + let output = self.run_ip_output(&args).await?; + if !output.success { + return Err(command_error(&args, &output, "listing namespaces")); + } + Ok(String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .any(|candidate| candidate == name)) + } + + async fn delete_namespace(&self, name: &str) -> Result<()> { + let args = vec!["netns".to_string(), "del".to_string(), name.to_string()]; + let output = self.run_ip_output(&args).await?; + if output.success { + return Ok(()); + } + let deletion = command_error(&args, &output, "cleaning up"); + match self.namespace_exists(name).await { + Ok(false) => Ok(()), + Ok(true) => Err(deletion), + Err(confirmation) => Err(BlazeError::BackendError { + msg: format!( + "{deletion}; cannot confirm namespace {name} was removed: {confirmation}" + ), + }), + } + } + + async fn fail_setup( + &self, + network: &NetworkSlot, + original: BlazeError, + ) -> std::result::Result { + match self.cleanup_commands(network).await { + Ok(()) => { + self.release(network.slot); + Err(NetworkCreateError::clean(original)) + } + Err(cleanup) => Err(NetworkCreateError::with_residual( + BlazeError::BackendError { + msg: format!( + "network setup failed ({original}); cleanup failed ({cleanup}); slot {} retained", + network.slot + ), + }, + network.clone(), + )), + } + } + + async fn run_in_namespace(&self, netns: &str, command: &[String]) -> Result<()> { + let mut args = vec!["netns".to_string(), "exec".to_string(), netns.to_string()]; + args.extend_from_slice(command); + self.run_ip(&args).await + } + + async fn run_in_namespace_cleanup(&self, netns: &str, command: &[String]) -> Result<()> { + let mut args = vec!["netns".to_string(), "exec".to_string(), netns.to_string()]; + args.extend_from_slice(command); + self.run_ip_cleanup(&args).await + } + + async fn run_ip(&self, args: &[String]) -> Result<()> { + let output = self.run_ip_output(args).await?; + if output.success { + return Ok(()); + } + Err(command_error(args, &output, "running command")) + } + + async fn run_ip_cleanup(&self, args: &[String]) -> Result<()> { + let output = self.run_ip_output(args).await?; + if output.success { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr); + if [ + "Cannot find device", + "No such file", + "Invalid \"netns\" value", + ] + .iter() + .any(|marker| stderr.contains(marker)) + { + return Ok(()); + } + Err(command_error(args, &output, "cleaning up")) + } + + async fn run_ip_output(&self, args: &[String]) -> Result { + self.runner.output(args, self.command_timeout).await + } +} + +fn command_error(args: &[String], output: &IpOutput, context: &str) -> BlazeError { + let stderr = String::from_utf8_lossy(&output.stderr); + let stderr = stderr.chars().take(4096).collect::(); + BlazeError::BackendError { + msg: format!( + "ip {} failed while {context} ({}): {}", + args.join(" "), + output.status, + stderr.trim() + ), + } +} + +#[cfg(unix)] +async fn acquire_host_guard(path: &Path, timeout: Duration) -> Result { + use std::os::fd::AsRawFd; + use std::os::unix::fs::OpenOptionsExt; + + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let deadline = Instant::now() + timeout; + loop { + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .mode(0o600) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW) + .open(path)?; + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) } == 0 { + return Ok(HostNetworkGuard { _file: Some(file) }); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::EAGAIN) + && error.raw_os_error() != Some(libc::EWOULDBLOCK) + { + return Err(error.into()); + } + if Instant::now() >= deadline { + return Err(BlazeError::BackendError { + msg: format!( + "timed out acquiring host network allocation lock {}", + path.display() + ), + }); + } + tokio::time::sleep(HOST_LOCK_RETRY).await; + } +} + +#[cfg(not(unix))] +async fn acquire_host_guard(_path: &Path, _timeout: Duration) -> Result { + Ok(HostNetworkGuard {}) +} + +#[cfg(target_os = "linux")] +fn executable_in_path(name: &str) -> bool { + let Some(path) = std::env::var_os("PATH") else { + return false; + }; + std::env::split_paths(&path).any(|directory| { + let candidate = directory.join(name); + if !candidate.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(candidate) + .map(|metadata| metadata.permissions().mode() & 0o111 != 0) + .unwrap_or(false) + } + #[cfg(not(unix))] + { + true + } + }) +} + +fn parse_existing_slots(stdout: &[u8]) -> HashSet { + String::from_utf8_lossy(stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|name| name.strip_prefix("blz-ns-")) + .filter_map(|suffix| suffix.split('-').next()) + .filter_map(|slot| slot.parse::().ok()) + .filter(|slot| *slot < NET_MAX_SLOT) + .collect() +} + +fn parse_existing_networks(stdout: &[u8]) -> Vec { + String::from_utf8_lossy(stdout) + .lines() + .filter_map(|line| line.split_whitespace().next()) + .filter_map(|name| name.strip_prefix("blz-ns-")) + .filter_map(|suffix| suffix.split_once('-')) + .filter_map(|(slot, owner)| { + Some(network_slot( + slot.parse::().ok()?, + owner.parse::().ok()?, + )) + }) + .filter(|network| network.slot < NET_MAX_SLOT) + .collect() +} + +fn network_slot(slot: usize, owner: Uuid) -> NetworkSlot { + NetworkSlot { + slot, + owner, + netns: format!("blz-ns-{slot}-{owner}"), + tap_name: "tap0".to_string(), + veth_host: format!("blz-veth-{slot}"), + veth_peer: format!("blz-vpeer-{slot}"), + } +} + +#[cfg(test)] +pub(super) fn test_network_slot(slot: usize) -> NetworkSlot { + network_slot(slot, Uuid::from_u128(1)) +} + +fn veth_ips(slot: usize) -> (String, String) { + let base = NET_VETH_BASE + slot * 4; + let third = (base >> 8) & 0xff; + ( + format!("169.254.{third}.{}", (base & 0xff) + 1), + format!("169.254.{third}.{}", (base & 0xff) + 2), + ) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + + use super::*; + + #[test] + fn allocator_is_unique_and_recycles() { + let manager = NetworkManager::default(); + let first = manager.allocate(&HashSet::new()).expect("first"); + let second = manager.allocate(&HashSet::new()).expect("second"); + assert_ne!(first, second); + manager.release(first); + for _ in 0..NET_MAX_SLOT { + if manager.allocate(&HashSet::new()).expect("slot") == first { + return; + } + } + panic!("released slot was not recycled"); + } + + #[test] + fn addresses_follow_the_slot_layout() { + assert_eq!( + veth_ips(0), + ("169.254.0.5".to_string(), "169.254.0.6".to_string()) + ); + assert_eq!( + veth_ips(63), + ("169.254.1.1".to_string(), "169.254.1.2".to_string()) + ); + } + + #[test] + fn existing_slot_parser_ignores_unrelated_and_invalid_names() { + let slots = parse_existing_slots( + b"blz-ns-0-00000000-0000-0000-0000-000000000001\n\ + blz-ns-17 (id: 2)\nunrelated\nblz-ns-nope\nblz-ns-16383\n", + ); + assert_eq!(slots, HashSet::from([0, 17])); + } + + #[cfg(unix)] + #[tokio::test] + async fn host_lock_serializes_independent_network_managers() { + let temp = tempfile::tempdir().expect("temp"); + let lock = temp.path().join("network.lock"); + let first = + NetworkManager::with_runner_and_lock(Arc::new(FakeIpRunner::default()), lock.clone()); + let second = NetworkManager::with_runner_and_lock(Arc::new(FakeIpRunner::default()), lock); + + let first_guard = first.acquire_host_guard().await.expect("first lock"); + let blocked = + tokio::time::timeout(Duration::from_millis(50), second.acquire_host_guard()).await; + assert!( + blocked.is_err(), + "a second daemon must not allocate while the host lock is held" + ); + + drop(first_guard); + second + .acquire_host_guard() + .await + .expect("lock becomes available"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn network_probe_checks_tools_privileges_and_ip_access() { + let missing_tool = Arc::new(FakeIpRunner::without_tool("iptables")); + let manager = NetworkManager::with_runner(missing_tool.clone()); + assert!(!manager.probe().await.expect("missing tool probe")); + assert!(missing_tool.calls().is_empty()); + + let unprivileged = Arc::new(FakeIpRunner::without_admin()); + let manager = NetworkManager::with_runner(unprivileged.clone()); + assert!(!manager.probe().await.expect("privilege probe")); + assert!(unprivileged.calls().is_empty()); + + let available = Arc::new(FakeIpRunner::with_responses([success(b"")])); + let manager = NetworkManager::with_runner(available.clone()); + assert!(manager.probe().await.expect("ready probe")); + assert_eq!( + available.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + + let inaccessible = Arc::new(FakeIpRunner::with_responses([failure( + "network namespace access denied", + )])); + let manager = NetworkManager::with_runner(inaccessible); + assert!(!manager.probe().await.expect("inaccessible probe")); + } + + #[tokio::test] + async fn create_skips_namespaces_owned_by_another_process() { + let runner = Arc::new(FakeIpRunner::with_responses([success( + b"blz-ns-0 (id: 4)\n", + )])); + let manager = NetworkManager::with_runner(runner.clone()); + + let owner = Uuid::from_u128(1); + let slot = manager + .create(owner, |_| Ok(())) + .await + .expect("create slot"); + + assert_eq!(slot.slot, 1); + let calls = runner.calls(); + assert!( + calls + .iter() + .any(|args| args == &["netns", "add", test_network_slot(1).netns()]) + ); + assert!( + !calls + .iter() + .any(|args| args == &["netns", "del", "blz-ns-0"]) + ); + } + + #[tokio::test] + async fn create_retries_when_namespace_appears_during_allocation() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + failure("namespace already exists"), + success(b"blz-ns-0\n"), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + let owner = Uuid::from_u128(1); + let slot = manager + .create(owner, |_| Ok(())) + .await + .expect("create slot"); + + assert_eq!(slot.slot, 1); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "add", test_network_slot(1).netns()]) + ); + } + + #[tokio::test] + async fn failed_veth_creation_only_removes_new_namespace() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + success(b""), + failure("veth already exists"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + manager + .create(Uuid::from_u128(1), |_| Ok(())) + .await + .expect_err("veth creation must fail"); + + let calls = runner.calls(); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", test_network_slot(0).netns()]) + ); + assert!( + !calls + .iter() + .any(|args| args == &["link", "del", "blz-veth-0"]) + ); + } + + #[tokio::test] + async fn failed_cleanup_returns_the_residual_slot_owner() { + let runner = Arc::new(FakeIpRunner::with_responses([ + success(b""), + success(b""), + success(b""), + failure("host address failed"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + failure("delete peer failed"), + success(b"blz-ns-0-00000000-0000-0000-0000-000000000001\n"), + success(b""), + success(b""), + ])); + let manager = NetworkManager::with_runner(runner.clone()); + + let failure = manager + .create(Uuid::from_u128(1), |_| Ok(())) + .await + .expect_err("setup must fail"); + let (source, residual) = failure.into_parts(); + let residual = residual.expect("cleanup failure must retain the slot"); + + assert!(source.to_string().contains("slot 0 retained")); + assert_eq!(residual, test_network_slot(0)); + manager + .destroy(&residual) + .await + .expect("a later cleanup can release the retained slot"); + let calls = runner.calls(); + assert!(calls.iter().any(|args| { + args == &[ + "netns", + "exec", + test_network_slot(0).netns(), + "ip", + "link", + "del", + "blz-vpeer-0", + ] + })); + assert!( + calls + .iter() + .any(|args| args == &["netns", "del", test_network_slot(0).netns()]) + ); + } + + #[tokio::test] + async fn namespace_delete_failure_retains_a_present_slot() { + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(FakeIpRunner::with_responses([ + success(namespace.as_bytes()), + success(b""), + failure("Cannot remove namespace file: Permission denied"), + success(namespace.as_bytes()), + ])); + let manager = NetworkManager::with_runner(runner); + manager + .state + .lock() + .expect("state lock") + .used + .insert(network.slot); + + let error = manager + .destroy(&network) + .await + .expect_err("present namespace must retain its slot"); + + assert!(error.to_string().contains("Permission denied")); + assert!( + manager + .state + .lock() + .expect("state lock") + .used + .contains(&network.slot) + ); + } + + #[tokio::test] + async fn namespace_delete_failure_accepts_confirmed_absence() { + let network = test_network_slot(0); + let namespace = format!("{}\n", network.netns()); + let runner = Arc::new(FakeIpRunner::with_responses([ + success(namespace.as_bytes()), + success(b""), + failure("Cannot remove namespace file: already removed"), + success(b""), + ])); + let manager = NetworkManager::with_runner(runner); + manager + .state + .lock() + .expect("state lock") + .used + .insert(network.slot); + + manager + .destroy(&network) + .await + .expect("confirmed absence completes cleanup"); + + assert!( + !manager + .state + .lock() + .expect("state lock") + .used + .contains(&network.slot) + ); + } + + struct FakeIpRunner { + responses: Mutex>, + calls: Mutex>>, + #[cfg(target_os = "linux")] + unavailable_tool: Option, + #[cfg(target_os = "linux")] + network_admin: bool, + } + + impl Default for FakeIpRunner { + fn default() -> Self { + Self { + responses: Mutex::new(VecDeque::new()), + calls: Mutex::new(Vec::new()), + #[cfg(target_os = "linux")] + unavailable_tool: None, + #[cfg(target_os = "linux")] + network_admin: true, + } + } + } + + impl FakeIpRunner { + fn with_responses(responses: [IpOutput; N]) -> Self { + Self { + responses: Mutex::new(responses.into()), + calls: Mutex::new(Vec::new()), + #[cfg(target_os = "linux")] + unavailable_tool: None, + #[cfg(target_os = "linux")] + network_admin: true, + } + } + + #[cfg(target_os = "linux")] + fn without_tool(tool: &str) -> Self { + Self { + unavailable_tool: Some(tool.to_string()), + ..Self::default() + } + } + + #[cfg(target_os = "linux")] + fn without_admin() -> Self { + Self { + network_admin: false, + ..Self::default() + } + } + + fn calls(&self) -> Vec> { + self.calls.lock().expect("calls lock").clone() + } + } + + #[async_trait] + impl IpCommandRunner for FakeIpRunner { + async fn output(&self, args: &[String], _timeout: Duration) -> Result { + self.calls.lock().expect("calls lock").push(args.to_vec()); + Ok(self + .responses + .lock() + .expect("responses lock") + .pop_front() + .unwrap_or_else(|| success(b""))) + } + + #[cfg(target_os = "linux")] + fn executable_in_path(&self, name: &str) -> bool { + self.unavailable_tool.as_deref() != Some(name) + } + + #[cfg(target_os = "linux")] + fn has_network_admin(&self) -> bool { + self.network_admin + } + } + + fn success(stdout: &[u8]) -> IpOutput { + IpOutput { + success: true, + status: "exit status: 0".to_string(), + stdout: stdout.to_vec(), + stderr: Vec::new(), + } + } + + fn failure(stderr: &str) -> IpOutput { + IpOutput { + success: false, + status: "exit status: 1".to_string(), + stdout: Vec::new(), + stderr: stderr.as_bytes().to_vec(), + } + } +} diff --git a/src/blaze/examples/policies/agent-rl.toml b/src/blaze/examples/policies/agent-rl.toml index 8fe5e79695..234cd0d518 100644 --- a/src/blaze/examples/policies/agent-rl.toml +++ b/src/blaze/examples/policies/agent-rl.toml @@ -118,6 +118,7 @@ memory = "4G" [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" enable_vsock = true # parsed but not yet wired (Phase 2+) +enable_network = false # opt in to a per-sandbox netns, tap, veth, and NAT serial_log = false # --------------------------------------------------------------------------- diff --git a/src/blaze/examples/policies/agent-tool.toml b/src/blaze/examples/policies/agent-tool.toml index 1d77a9b410..60abea8ae8 100644 --- a/src/blaze/examples/policies/agent-tool.toml +++ b/src/blaze/examples/policies/agent-tool.toml @@ -106,6 +106,7 @@ memory = "1G" [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" enable_vsock = false # parsed but not yet wired (Phase 2+) +enable_network = false # opt in to a per-sandbox netns, tap, veth, and NAT memory = "2G" vcpus = 2 serial_log = false From 0a82000b62663c647efd3d7ff42738e9862ae7f9 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 16:13:56 +0800 Subject: [PATCH 06/40] docs(blaze): explain isolated VM networking This documents the Firecracker policy switch that enables a dedicated network slot, the host-wide allocation coordination, and the startup checks performed when a loaded policy requests the capability. It also records the ownership boundary: namespace, link, and tap cleanup follow the backend process, while upstream routing and DNS remain host-operator responsibilities and networking stays disabled by default. This lets reviewers and operators see what sandbox creation gains and what the daemon deliberately does not configure. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 16 +++++++++++++++- src/blaze/README_zh.md | 11 +++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/blaze/README.md b/src/blaze/README.md index 8f4b0c00a1..7050dc4373 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -19,6 +19,7 @@ Designed as the per-host agent for E2B-style orchestrator platforms. - **Kernel hook registry** — state tracking for pre/post hooks - **Prometheus metrics** — request counts, instance gauges, pool sizes - **Spawners** — FirecrackerSpawner, BubblewrapSpawner, MockSpawner +- **Optional VM networking** — isolated namespace, tap, veth, and NAT per Firecracker VM ## Quick Start @@ -76,8 +77,20 @@ memory = "512Mi" [backend.firecracker] vcpus = 4 # overrides [vm].vcpus for Firecracker only memory = "1Gi" # overrides [vm].memory for Firecracker only +enable_network = false ``` +Set `enable_network = true` to create an isolated network slot for each +Firecracker VM. Explicit sandbox destroy and compensated startup failure remove +the namespace, tap, and veth after process termination. A destroy retried after +a daemon restart can reconstruct the recorded slot; there is no background +cleanup scan. Slot creation and deletion use a host-wide lock so independent +daemon processes cannot allocate the same host device names concurrently. +When a loaded Firecracker policy enables this option, backend probing also +checks the required commands and host privileges. The checks are skipped when +networking is disabled. Upstream routing and DNS remain host operator +responsibilities. + ### Storage Configuration The `[storage]` section controls the sandbox storage backend: @@ -146,6 +159,7 @@ src/blaze/ - Rust 1.88+ (see `src/blaze/rust-toolchain.toml`) - Linux host with root privileges for sandbox backends +- `ip`, `iptables`, `sysctl`, and network namespace privileges when VM + networking is enabled ## License - diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index fddef048c4..1d34b946fe 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -18,6 +18,7 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **内核 hook 注册** — 前/后置 hook 状态追踪 - **Prometheus 指标** — 请求计数、实例 gauge、池大小 - **Spawner 后端** — FirecrackerSpawner、BubblewrapSpawner、MockSpawner +- **可选 VM 网络** — 每台 Firecracker VM 独立使用 netns、tap、veth 和 NAT ## 快速开始 @@ -75,8 +76,17 @@ memory = "512Mi" [backend.firecracker] vcpus = 4 # 仅对 Firecracker 覆盖 [vm].vcpus memory = "1Gi" # 仅对 Firecracker 覆盖 [vm].memory +enable_network = false ``` +设置 `enable_network = true` 后，每台 Firecracker VM 会获得独立的网络 +slot。显式销毁 sandbox 和启动失败补偿会在进程确认终止后删除对应的 netns、 +tap 和 veth。daemon 重启后再次销毁时可以根据记录恢复清理，但不会在后台 +自动扫描。slot 创建和删除使用主机级锁，避免多个 daemon 同时分配相同的主机 +设备名。加载的 Firecracker 策略启用该选项时，backend probe 还会检查所需 +命令和主机权限；网络关闭时跳过这些检查。上游路由和 DNS 仍由主机运维方 +配置。 + ### 存储配置 `[storage]` 部分控制 sandbox 存储后端： @@ -145,6 +155,7 @@ src/blaze/ - Rust 1.88+（参见 `src/blaze/rust-toolchain.toml`） - 具有 root 权限的 Linux 主机（sandbox 后端需要） +- 启用 VM 网络时需要 `ip`、`iptables`、`sysctl` 和 netns 管理权限 ## 许可证 From b2165798f6fcd60c66e48fd3f9cc9468a2c51458 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 12:01:15 +0800 Subject: [PATCH 07/40] fix(blaze): stop on corrupt sandbox state This enables daemon startup to stop when an owned state directory cannot be loaded or its record names a different sandbox. Persisted runtime ownership is never hidden under an invalid or misleading key. UUID-named directories must contain readable state whose embedded ID matches the directory name, and ServerState::build propagates any scan failure. A failed first state publication removes only the empty directory it created; a directory containing another owned artifact remains in place and fails closed. One damaged owned record now prevents API startup until it is repaired. This avoids serving an incomplete inventory while also preventing an ordinary first-write failure from leaving an empty directory that blocks every later startup. Fixes: 1f0cfac49 Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/lifecycle.rs | 65 ++++++++++++-- src/blaze/crates/blazed/src/api.rs | 69 ++++++++------- src/blaze/crates/blazed/src/daemon.rs | 2 +- src/blaze/crates/blazed/src/state.rs | 90 +++++++++++++++----- 4 files changed, 171 insertions(+), 55 deletions(-) diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index cecf6bbb2a..4fdfac9698 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -149,14 +149,37 @@ impl SandboxInstance { /// Persist this instance to `{state_dir}/{id}/state.json`. Atomic /// rename via `state.json.tmp` to avoid torn reads on daemon restart. pub fn persist(&self, state_dir: &Path) -> Result<()> { + self.persist_with(state_dir, |tmp_path, final_path, json| { + fs::write(tmp_path, json)?; + fs::rename(tmp_path, final_path)?; + Ok(()) + }) + } + + fn persist_with(&self, state_dir: &Path, publish: F) -> Result<()> + where + F: FnOnce(&Path, &Path, &[u8]) -> Result<()>, + { let dir = state_dir.join(self.id.to_string()); - fs::create_dir_all(&dir)?; + let json = serde_json::to_vec_pretty(self)?; + fs::create_dir_all(state_dir)?; + let created_owner_dir = match fs::create_dir(&dir) { + Ok(()) => true, + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => false, + Err(error) => return Err(error.into()), + }; let final_path = dir.join("state.json"); let tmp_path = dir.join("state.json.tmp"); - let json = serde_json::to_vec_pretty(self)?; - fs::write(&tmp_path, &json)?; - fs::rename(&tmp_path, &final_path)?; - Ok(()) + let result = publish(&tmp_path, &final_path, &json); + if result.is_err() { + let _ = fs::remove_file(&tmp_path); + if created_owner_dir { + // Remove only an empty directory. If another owned artifact + // appeared, retain the directory so startup fails closed. + let _ = fs::remove_dir(&dir); + } + } + result } /// Reload an instance previously persisted via [`Self::persist`]. @@ -278,4 +301,36 @@ mod tests { assert_eq!(loaded.state, SandboxState::Creating); assert_eq!(loaded.policy_name, inst.policy_name); } + + #[test] + fn failed_first_persist_removes_an_empty_owner_directory() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + + let error = instance + .persist_with(tmp.path(), |_tmp_path, _final_path, _json| { + Err(std::io::Error::other("injected publication failure").into()) + }) + .expect_err("publication must fail"); + + assert!(error.to_string().contains("injected publication failure")); + assert!(!tmp.path().join(instance.id.to_string()).exists()); + } + + #[test] + fn failed_persist_preserves_a_directory_with_owned_artifacts() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let owner_dir = tmp.path().join(instance.id.to_string()); + std::fs::create_dir_all(&owner_dir).expect("owner dir"); + std::fs::write(owner_dir.join("backend.pid"), b"owner").expect("owner marker"); + + instance + .persist_with(tmp.path(), |_tmp_path, _final_path, _json| { + Err(std::io::Error::other("injected publication failure").into()) + }) + .expect_err("publication must fail"); + + assert!(owner_dir.join("backend.pid").exists()); + } } diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 2fd473384c..66e03cd428 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -1404,16 +1404,19 @@ mod tests { active_backend: BackendKind, storage: Arc, ) -> Arc { - Arc::new(ServerState::build( - config, - PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), - registry, - active_backend, - storage, - )) + Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(vec![policy]), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + registry, + active_backend, + storage, + ) + .expect("build server state"), + ) } #[cfg(feature = "test-failpoints")] @@ -1693,16 +1696,19 @@ mod tests { let _ = std::fs::create_dir_all(&storage_dir); let storage: Arc = Arc::new(FileStorageProvider::new(storage_dir)); - let state = Arc::new(ServerState::build( - config, - engine, - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), - spawners(BackendKind::Firecracker, spawner), - BackendKind::Firecracker, - storage, - )); + let state = Arc::new( + ServerState::build( + config, + engine, + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Firecracker, spawner), + BackendKind::Firecracker, + storage, + ) + .expect("build server state"), + ); // Create instance request for AgentRl workload. let req_body = serde_json::to_vec(&serde_json::json!({ @@ -1863,16 +1869,19 @@ mod tests { config.storage.images_dir.clone(), config.storage.instances_dir.clone(), )); - let state = Arc::new(ServerState::build( - config, - PolicyEngine::with_policies(vec![policy]), - PoolManager::new(), - TemplateRegistry::new(), - HookRegistry::new(), - spawners(BackendKind::Mock, Arc::new(MockSpawner)), - BackendKind::Mock, - storage, - )); + let state = Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(vec![policy]), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ) + .expect("build server state"), + ); let request = serde_json::to_vec(&json!({ "workload_class": "agent-tool", "image_digest": "sha256:warm-validation" diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index a9616003f3..1c80d6a7b0 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -77,7 +77,7 @@ pub async fn run(config_path: &Path) -> Result<()> { spawners, active_backend, storage, - )); + )?); if socket_path.exists() { std::fs::remove_file(&socket_path)?; diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index 0dcece9a63..422ef08ca8 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -20,7 +20,7 @@ use blaze_core::template::TemplateRegistry; use tokio::sync::Mutex as AsyncMutex; use uuid::Uuid; -use crate::error::Result; +use crate::error::{BlazeDaemonError, Result}; use crate::metrics::Metrics; use crate::spawner::{DynBackendInstance, DynSpawner, SpawnerRegistry}; @@ -47,8 +47,7 @@ pub struct ServerState { impl ServerState { /// Build a server state, scanning `state_dir` to repopulate the - /// `instances` map from previous runs (best-effort; corrupt entries - /// are skipped with a warning). + /// `instances` map from previous runs. #[allow(clippy::too_many_arguments)] pub fn build( config: DaemonConfig, @@ -59,19 +58,16 @@ impl ServerState { spawners: SpawnerRegistry, active_backend: BackendKind, storage: Arc, - ) -> Self { + ) -> Result { let state_dir = config.daemon.state_dir.clone(); - let instances = scan_state_dir(&state_dir).unwrap_or_else(|err| { - tracing::warn!(error = %err, "failed to scan state_dir, starting empty"); - HashMap::new() - }); + let instances = scan_state_dir(&state_dir)?; let operation_locks = instances .keys() .copied() .map(|id| (id, Arc::new(AsyncMutex::new(())))) .collect(); - Self { + Ok(Self { config: Mutex::new(config), policy: Mutex::new(policy), pool: Mutex::new(pool), @@ -85,7 +81,7 @@ impl ServerState { storage, state_dir, metrics: Metrics::new(), - } + }) } /// Return the async operation lock that serializes one sandbox mutation. @@ -109,8 +105,10 @@ impl ServerState { } } -/// Best-effort: walk `{state_dir}//state.json` and rebuild the -/// instance map. Used both at boot and (in the future) by `daemon doctor`. +/// Walk `{state_dir}//state.json` and rebuild the instance map. +/// +/// A valid UUID directory is owned lifecycle state. If its record cannot be +/// loaded, startup must stop rather than hide resources from later cleanup. fn scan_state_dir(state_dir: &Path) -> Result> { let mut out = HashMap::new(); if !state_dir.exists() { @@ -128,15 +126,69 @@ fn scan_state_dir(state_dir: &Path) -> Result> { let Ok(id) = Uuid::parse_str(name_str) else { continue; }; - match SandboxInstance::load(state_dir, id) { - Ok(inst) => { - out.insert(id, inst); - } - Err(err) => { - tracing::warn!(instance = %id, error = %err, "skipping corrupt instance state"); - } + let instance = SandboxInstance::load(state_dir, id).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot load persisted instance {id}: {error}" + )) + })?; + if instance.id != id { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "persisted instance id {} does not match owned directory {id}", + instance.id + ))); } + out.insert(id, instance); } tracing::info!(instances = out.len(), "rehydrated instances from state_dir"); Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scan_rejects_corrupt_owned_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let id = Uuid::new_v4(); + let instance_dir = temp.path().join(id.to_string()); + std::fs::create_dir_all(&instance_dir).expect("instance dir"); + std::fs::write(instance_dir.join("state.json"), b"{not-json").expect("state"); + + let error = scan_state_dir(temp.path()).expect_err("corrupt state must stop startup"); + + assert!(matches!( + error, + BlazeDaemonError::RecoveryRequired(message) + if message.contains(&id.to_string()) + && message.contains("cannot load persisted instance") + )); + } + #[test] + fn scan_rejects_state_owned_by_a_different_directory() { + let temp = tempfile::tempdir().expect("tempdir"); + let instance = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:mismatched-id".into(), + blaze_core::lifecycle::StartPath::Cold, + "test".into(), + ); + instance.persist(temp.path()).expect("persist state"); + let directory_id = Uuid::new_v4(); + std::fs::rename( + temp.path().join(instance.id.to_string()), + temp.path().join(directory_id.to_string()), + ) + .expect("rename directory"); + + let error = scan_state_dir(temp.path()).expect_err("mismatched state must stop startup"); + + assert!(matches!( + error, + BlazeDaemonError::RecoveryRequired(message) + if message.contains(&instance.id.to_string()) + && message.contains(&directory_id.to_string()) + )); + } +} From 42ff6dd4f49e3b22d22b97b8dc8635c4acc6e042 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 12:01:15 +0800 Subject: [PATCH 08/40] fix(blaze): reclaim inactive operation locks This enables a long-running daemon to serialize per-instance mutations without retaining one lock entry for every UUID it has seen. The registry stores weak references and prunes entries with no holders or waiters while holding the registry mutex. Concurrent callers for a live UUID still share one async lock. Completed instances therefore leave no permanent strong reference in the registry. Fixes: c33b495450 Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/state.rs | 55 +++++++++++++++++++--------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index 422ef08ca8..70192aabf8 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, Weak}; use blaze_core::backend::BackendKind; use blaze_core::config::DaemonConfig; @@ -34,7 +34,7 @@ pub struct ServerState { pub hook: Mutex, pub instances: Mutex>, pub backend_instances: Mutex>, - operation_locks: Mutex>>>, + operation_locks: Mutex>>>, pub spawners: SpawnerRegistry, /// The backend kind that `build_spawner` actually probed and selected. /// API handlers use this to constrain availability to the single active @@ -61,11 +61,6 @@ impl ServerState { ) -> Result { let state_dir = config.daemon.state_dir.clone(); let instances = scan_state_dir(&state_dir)?; - let operation_locks = instances - .keys() - .copied() - .map(|id| (id, Arc::new(AsyncMutex::new(())))) - .collect(); Ok(Self { config: Mutex::new(config), @@ -75,7 +70,7 @@ impl ServerState { hook: Mutex::new(hook), instances: Mutex::new(instances), backend_instances: Mutex::new(HashMap::new()), - operation_locks: Mutex::new(operation_locks), + operation_locks: Mutex::new(HashMap::new()), spawners, active_backend, storage, @@ -87,15 +82,8 @@ impl ServerState { /// Return the async operation lock that serializes one sandbox mutation. pub fn operation_lock(&self, id: Uuid) -> Arc> { match self.operation_locks.lock() { - Ok(mut locks) => locks - .entry(id) - .or_insert_with(|| Arc::new(AsyncMutex::new(()))) - .clone(), - Err(poisoned) => poisoned - .into_inner() - .entry(id) - .or_insert_with(|| Arc::new(AsyncMutex::new(()))) - .clone(), + Ok(mut locks) => operation_lock(&mut locks, id), + Err(poisoned) => operation_lock(&mut poisoned.into_inner(), id), } } @@ -105,6 +93,20 @@ impl ServerState { } } +fn operation_lock( + locks: &mut HashMap>>, + id: Uuid, +) -> Arc> { + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&id).and_then(Weak::upgrade) { + return lock; + } + + let lock = Arc::new(AsyncMutex::new(())); + locks.insert(id, Arc::downgrade(&lock)); + lock +} + /// Walk `{state_dir}//state.json` and rebuild the instance map. /// /// A valid UUID directory is owned lifecycle state. If its record cannot be @@ -191,4 +193,23 @@ mod tests { && message.contains(&directory_id.to_string()) )); } + + #[test] + fn operation_locks_reuse_live_entries_and_prune_dead_ones() { + let mut locks = HashMap::new(); + let id = Uuid::new_v4(); + let first = operation_lock(&mut locks, id); + let second = operation_lock(&mut locks, id); + + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(locks.len(), 1); + + drop(first); + drop(second); + for _ in 0..256 { + drop(operation_lock(&mut locks, Uuid::new_v4())); + } + + assert_eq!(locks.len(), 1); + } } From 0704ed9a00ee444044913ec037a2b9d8a2f54214 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 15:40:10 +0800 Subject: [PATCH 09/40] fix(blaze): drain daemon connection tasks This keeps accepted HTTP connections under daemon ownership so shutdown no longer returns while request handlers are still using runtime resources. The accept loop tracks and reaps connection tasks. On termination it closes listeners, asks HTTP/1 connections to finish, waits for a bounded drain window, and aborts and joins any remaining tasks. This enables shutdown to establish a deterministic boundary before process exit while still preventing an idle or stuck client from blocking the daemon indefinitely. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/daemon.rs | 258 ++++++++++++++++++++++++-- 1 file changed, 242 insertions(+), 16 deletions(-) diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 1c80d6a7b0..630c5bbc12 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -1,8 +1,10 @@ // SPDX-License-Identifier: Apache-2.0 //! Daemon runtime: bind UDS, accept connections, wire signal handlers. +use std::future::Future; use std::path::Path; use std::sync::Arc; +use std::time::Duration; use blaze_core::backend::BackendKind; use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode}; @@ -11,13 +13,13 @@ use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use blaze_core::template::TemplateRegistry; -use http_body_util::Full; -use hyper::body::Bytes; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; use tokio::net::{TcpListener, UnixListener}; use tokio::signal::unix::{SignalKind, signal}; +use tokio::sync::watch; +use tokio::task::{JoinError, JoinSet}; use crate::api; use crate::error::{BlazeDaemonError, Result}; @@ -26,6 +28,99 @@ use crate::spawner::{ }; use crate::state::ServerState; +const CONNECTION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); + +struct ConnectionSupervisor { + shutdown: watch::Sender, + tasks: JoinSet<()>, +} + +impl ConnectionSupervisor { + fn new() -> Self { + let (shutdown, _) = watch::channel(false); + Self { + shutdown, + tasks: JoinSet::new(), + } + } + + fn is_empty(&self) -> bool { + self.tasks.is_empty() + } + + fn spawn(&mut self, io: TokioIo, state: Arc) + where + I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, + { + let shutdown = self.shutdown.subscribe(); + self.spawn_task(serve_connection(io, state, shutdown)); + } + + fn spawn_task(&mut self, task: F) + where + F: Future + Send + 'static, + { + self.tasks.spawn(task); + } + + async fn join_next(&mut self) -> Option> { + self.tasks.join_next().await + } + + async fn shutdown(mut self, grace: Duration) -> Result<()> { + self.shutdown.send_replace(true); + let mut task_failed = false; + let completed = tokio::time::timeout(grace, async { + while let Some(result) = self.tasks.join_next().await { + task_failed |= report_connection_result(result); + } + }) + .await; + + if completed.is_err() { + let remaining = self.tasks.len(); + tracing::warn!( + remaining, + timeout_secs = grace.as_secs(), + "connection drain timed out; aborting remaining tasks" + ); + self.tasks.abort_all(); + while let Some(result) = self.tasks.join_next().await { + if let Err(error) = result + && !error.is_cancelled() + { + report_connection_result(Err(error)); + } + } + return Err(BlazeDaemonError::Internal(format!( + "connection drain timed out after {} seconds; aborted {remaining} task(s)", + grace.as_secs() + ))); + } + + if task_failed { + return Err(BlazeDaemonError::Internal( + "one or more connection tasks failed during shutdown".to_string(), + )); + } + Ok(()) + } +} + +fn report_connection_result(result: std::result::Result<(), JoinError>) -> bool { + match result { + Ok(()) => false, + Err(error) if error.is_cancelled() => { + tracing::debug!("connection task cancelled"); + false + } + Err(error) => { + tracing::error!(%error, "connection task failed"); + true + } + } +} + /// Boot the daemon: load config + policies, prepare state directories, /// bind the API socket, and run the accept loop until SIGTERM/SIGINT. pub async fn run(config_path: &Path) -> Result<()> { @@ -223,9 +318,15 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc { + if let Some(completed) = completed { + report_connection_result(completed); + } + } res = uds.accept() => { let (stream, _peer) = match res { Ok(s) => s, @@ -234,7 +335,7 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc l.accept().await, None => std::future::pending().await }}, if tcp.is_some() => { let (stream, peer) = match res { @@ -245,7 +346,7 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc { tracing::info!("SIGHUP received: reloading policies"); @@ -264,24 +365,48 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc(io: TokioIo, state: Arc) -where +async fn serve_connection( + io: TokioIo, + state: Arc, + mut shutdown: watch::Receiver, +) where I: tokio::io::AsyncRead + tokio::io::AsyncWrite + Unpin + Send + 'static, { - tokio::spawn(async move { - let svc = service_fn(move |req| { - let state = state.clone(); - async move { api::handle(req, state).await } - }); - if let Err(err) = http1::Builder::new().serve_connection(io, svc).await { - tracing::debug!(?err, "connection closed with error"); - } - let _: Option> = None; + let svc = service_fn(move |req| { + let state = state.clone(); + async move { api::handle(req, state).await } }); + let connection = http1::Builder::new().serve_connection(io, svc); + tokio::pin!(connection); + let result = tokio::select! { + result = &mut connection => result, + _ = wait_for_shutdown(&mut shutdown) => { + connection.as_mut().graceful_shutdown(); + connection.await + } + }; + if let Err(error) = result { + tracing::debug!(%error, "connection closed with error"); + } +} + +async fn wait_for_shutdown(shutdown: &mut watch::Receiver) { + loop { + let stopping = *shutdown.borrow_and_update(); + if stopping { + return; + } + if shutdown.changed().await.is_err() { + return; + } + } } fn reload_policies(state: &Arc) -> Result<()> { @@ -304,3 +429,104 @@ fn reload_policies(state: &Arc) -> Result<()> { tracing::info!(policies = count, "policy engine reloaded via SIGHUP"); Ok(()) } + +#[cfg(test)] +mod tests { + use std::future; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + use tokio::sync::oneshot; + + use super::*; + + struct ActiveTask(Arc); + + impl Drop for ActiveTask { + fn drop(&mut self) { + self.0.store(false, Ordering::Release); + } + } + + #[tokio::test] + async fn shutdown_waits_for_inflight_connection_task() { + let mut connections = ConnectionSupervisor::new(); + let mut shutdown = connections.shutdown.subscribe(); + let (entered_tx, entered_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + connections.spawn_task(async move { + entered_tx.send(()).expect("signal task entry"); + shutdown + .wait_for(|stopping| *stopping) + .await + .expect("shutdown sender"); + release_rx.await.expect("release task"); + }); + entered_rx.await.expect("task entered"); + + let drain = tokio::spawn(connections.shutdown(Duration::from_secs(1))); + tokio::task::yield_now().await; + assert!(!drain.is_finished(), "drain returned before task completed"); + + release_tx.send(()).expect("release connection task"); + drain + .await + .expect("drain task") + .expect("graceful connection drain"); + } + + #[tokio::test] + async fn shutdown_notifies_idle_connection_task() { + let mut connections = ConnectionSupervisor::new(); + let mut shutdown = connections.shutdown.subscribe(); + connections.spawn_task(async move { + shutdown + .wait_for(|stopping| *stopping) + .await + .expect("shutdown sender"); + }); + + connections + .shutdown(Duration::from_secs(1)) + .await + .expect("idle task drained"); + } + + #[tokio::test] + async fn shutdown_timeout_aborts_and_joins_stuck_task() { + let mut connections = ConnectionSupervisor::new(); + let active = Arc::new(AtomicBool::new(true)); + let task_active = active.clone(); + let (entered_tx, entered_rx) = oneshot::channel(); + connections.spawn_task(async move { + let _active = ActiveTask(task_active); + entered_tx.send(()).expect("signal task entry"); + future::pending::<()>().await; + }); + entered_rx.await.expect("task entered"); + + let error = connections + .shutdown(Duration::from_millis(10)) + .await + .expect_err("stuck task must time out"); + + assert!(error.to_string().contains("connection drain timed out")); + assert!( + !active.load(Ordering::Acquire), + "aborted task was not joined" + ); + } + + #[tokio::test] + async fn completed_tasks_are_reaped_while_serving() { + let mut connections = ConnectionSupervisor::new(); + connections.spawn_task(async {}); + + let completed = connections + .join_next() + .await + .expect("one tracked connection"); + assert!(!report_connection_result(completed)); + assert!(connections.is_empty()); + } +} From 9ffc46e4e80eeb0cb52ce28c680108c9921f946a Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 17:22:42 +0800 Subject: [PATCH 10/40] fix(blaze): release runtimes during shutdown The daemon previously stopped after connection handling ended while tracked backend processes and provider slots could remain owned by the exiting process. After connection tasks drain, shutdown starts the existing destroy path for every non-terminal sandbox and retained backend owner. Cleanup attempts run concurrently under one shared work deadline, so a stalled owner cannot multiply stop latency or prevent independent owners from being released. Both shutdown stages cancel and join unfinished tasks before returning. The service manager provides the final stop deadline with cancellation headroom, so no cleanup task continues to access daemon state after an in-process stage finishes. This enables SIGTERM and SIGINT shutdown to release tracked runtime resources while keeping incomplete ownership recoverable and reporting observed cleanup failures. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/api.rs | 370 +++++++++++++++++++++++++- src/blaze/crates/blazed/src/daemon.rs | 95 ++++++- src/blaze/dist/blazed.service | 2 +- 3 files changed, 462 insertions(+), 5 deletions(-) diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 66e03cd428..307af8e1f8 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -5,10 +5,11 @@ //! than a router framework — the surface is small (~17 endpoints) and //! the cost of a fresh dependency outweighs the readability win. -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::convert::Infallible; use std::str::FromStr; use std::sync::Arc; +use std::time::Duration; use blaze_core::BlazeError; use blaze_core::backend::{BackendKind, BackendStatus, SpawnRequest, select_backend}; @@ -23,6 +24,8 @@ use hyper::header::CONTENT_TYPE; use hyper::{Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::json; +use tokio::task::JoinSet; +use tokio::time::Instant; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; @@ -1051,6 +1054,129 @@ async fn destroy_instance(state: &Arc, id: &str) -> Result, budget: Duration) -> Result<()> { + let mut ids = state + .instances + .lock() + .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? + .iter() + .filter_map(|(id, instance)| { + let clean_terminal = instance.state == SandboxState::Destroyed + && matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ); + (!clean_terminal).then_some(*id) + }) + .collect::>(); + ids.extend( + state + .backend_instances + .lock() + .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? + .keys() + .copied(), + ); + + let deadline = Instant::now() + budget; + let mut tasks = JoinSet::new(); + let mut task_owners = HashMap::new(); + for id in ids { + let state = state.clone(); + let task = + tasks.spawn(async move { destroy_instance(&state, &id.to_string()).await.map(|_| ()) }); + task_owners.insert(task.id(), id); + } + + let mut failures = BTreeMap::new(); + let mut deadline_expired = false; + while !tasks.is_empty() { + match tokio::time::timeout_at(deadline, tasks.join_next_with_id()).await { + Ok(Some(Ok((task_id, result)))) => { + let Some(id) = task_owners.remove(&task_id) else { + failures.insert( + format!("task-{task_id}"), + "cleanup result had no tracked sandbox owner".to_string(), + ); + continue; + }; + if let Err(error) = result { + failures.insert(id.to_string(), error.to_string()); + } + } + Ok(Some(Err(error))) => { + let key = task_owners + .remove(&error.id()) + .map_or_else(|| format!("task-{}", error.id()), |id| id.to_string()); + failures.insert(key, format!("cleanup task failed: {error}")); + } + Ok(None) => break, + Err(_) => { + deadline_expired = true; + break; + } + } + } + + if deadline_expired { + let mut timed_out_owners = task_owners.values().copied().collect::>(); + tasks.abort_all(); + while let Some(result) = tasks.join_next_with_id().await { + match result { + Ok((task_id, result)) => { + let Some(id) = task_owners.remove(&task_id) else { + failures.insert( + format!("task-{task_id}"), + "cleanup result had no tracked sandbox owner".to_string(), + ); + continue; + }; + timed_out_owners.remove(&id); + if let Err(error) = result { + failures.insert(id.to_string(), error.to_string()); + } + } + Err(error) => { + let owner = task_owners.remove(&error.id()); + let key = + owner.map_or_else(|| format!("task-{}", error.id()), |id| id.to_string()); + if !error.is_cancelled() { + if let Some(id) = owner { + timed_out_owners.remove(&id); + } + failures.insert(key, format!("cleanup task failed: {error}")); + } + } + } + } + for id in timed_out_owners { + failures.entry(id.to_string()).or_insert_with(|| { + format!("cleanup did not finish within the shared {budget:?} budget") + }); + } + } + + if failures.is_empty() { + Ok(()) + } else { + let failures = failures + .into_iter() + .map(|(owner, error)| format!("{owner}: {error}")) + .collect::>(); + Err(BlazeDaemonError::RecoveryRequired(format!( + "daemon shutdown left {} sandbox cleanup operation(s) incomplete: {}", + failures.len(), + failures.join("; ") + ))) + } +} + // --------------------------------------------------------------------------- // Pools // --------------------------------------------------------------------------- @@ -1314,6 +1440,7 @@ fn _hookkind_marker(_k: HookKind) {} #[cfg(test)] mod tests { use std::collections::HashMap; + use std::future; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -1579,6 +1706,105 @@ mod tests { } } + #[derive(Clone, Copy)] + enum ShutdownBehavior { + Complete, + Fail, + Stall, + } + + struct ShutdownOwner { + instance_id: Uuid, + behavior: ShutdownBehavior, + attempts: Arc, + active: Arc, + } + + struct ActiveCleanup(Arc); + + impl ActiveCleanup { + fn enter(active: Arc) -> Self { + active.fetch_add(1, Ordering::AcqRel); + Self(active) + } + } + + impl Drop for ActiveCleanup { + fn drop(&mut self) { + self.0.fetch_sub(1, Ordering::AcqRel); + } + } + + #[async_trait] + impl BackendInstance for ShutdownOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(None) + } + + async fn kill(&self) -> blaze_core::Result<()> { + self.attempts.fetch_add(1, Ordering::AcqRel); + let _active = ActiveCleanup::enter(self.active.clone()); + match self.behavior { + ShutdownBehavior::Complete => Ok(()), + ShutdownBehavior::Fail => Err(BlazeError::BackendError { + msg: format!("instance {} termination failed", self.instance_id), + }), + ShutdownBehavior::Stall => future::pending().await, + } + } + } + + async fn track_shutdown_owner( + state: &Arc, + behavior: ShutdownBehavior, + attempts: Arc, + active: Arc, + ) -> Uuid { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:shutdown-budget".into(), + StartPath::Cold, + "shutdown-budget-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance.persist(&state.state_dir).expect("persist"); + state + .storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 4096, + mem_size: 4096, + }) + .await + .expect("storage"); + + let id = instance.id; + state + .instances + .lock() + .expect("instances") + .insert(id, instance); + state.backend_instances.lock().expect("owners").insert( + id, + Arc::new(ShutdownOwner { + instance_id: id, + behavior, + attempts, + active, + }), + ); + id + } + struct PartialSpawnSpawner; #[async_trait] @@ -1975,6 +2201,148 @@ mod tests { assert!(observed.load(Ordering::Acquire)); } + #[tokio::test] + async fn daemon_shutdown_releases_tracked_runtime_resources() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"); + + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown cleanup"); + + assert_eq!( + state + .instances + .lock() + .expect("instances") + .get(&id) + .expect("instance") + .state, + SandboxState::Destroyed + ); + assert!( + !state + .backend_instances + .lock() + .expect("backend owners") + .contains_key(&id) + ); + assert!(!instances_dir.join(id.to_string()).exists()); + } + + #[tokio::test] + async fn daemon_shutdown_joins_all_owners_within_one_budget() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let attempts = Arc::new(AtomicUsize::new(0)); + let active = Arc::new(AtomicUsize::new(0)); + let complete = track_shutdown_owner( + &state, + ShutdownBehavior::Complete, + attempts.clone(), + active.clone(), + ) + .await; + let failed = track_shutdown_owner( + &state, + ShutdownBehavior::Fail, + attempts.clone(), + active.clone(), + ) + .await; + let stalled_a = track_shutdown_owner( + &state, + ShutdownBehavior::Stall, + attempts.clone(), + active.clone(), + ) + .await; + let stalled_b = track_shutdown_owner( + &state, + ShutdownBehavior::Stall, + attempts.clone(), + active.clone(), + ) + .await; + + let started = Instant::now(); + let error = shutdown_instances(&state, Duration::from_millis(40)) + .await + .expect_err("stalled cleanup must exhaust the shared budget"); + + assert!( + started.elapsed() < Duration::from_millis(400), + "cleanup did not quiesce promptly after the test deadline" + ); + assert_eq!( + attempts.load(Ordering::Acquire), + 4, + "every owner must receive a cleanup attempt" + ); + assert_eq!( + active.load(Ordering::Acquire), + 0, + "shutdown returned before every cleanup task became quiescent" + ); + let message = error.to_string(); + for id in [failed, stalled_a, stalled_b] { + assert!(message.contains(&id.to_string())); + assert!( + state + .backend_instances + .lock() + .expect("owners") + .contains_key(&id) + ); + assert!(instances_dir.join(id.to_string()).is_dir()); + } + assert_eq!( + state + .instances + .lock() + .expect("instances") + .get(&complete) + .expect("complete instance") + .state, + SandboxState::Destroyed + ); + assert!( + !state + .backend_instances + .lock() + .expect("owners") + .contains_key(&complete) + ); + assert!(!instances_dir.join(complete.to_string()).exists()); + } + #[tokio::test] async fn mock_fallback_uses_runtime_backend_for_warm_reuse() { let temp = tempfile::tempdir().expect("temp"); diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 630c5bbc12..3ab8c46707 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -28,7 +28,23 @@ use crate::spawner::{ }; use crate::state::ServerState; -const CONNECTION_DRAIN_TIMEOUT: Duration = Duration::from_secs(30); +#[derive(Clone, Copy)] +struct ShutdownBudget { + connection_drain: Duration, + runtime_cleanup: Duration, +} + +const SHUTDOWN_BUDGET: ShutdownBudget = ShutdownBudget { + // Preserve the existing client grace period. The supervisor aborts and + // joins handlers that do not finish before this deadline. + connection_drain: Duration::from_secs(30), + // Runtime cleanup starts for all owners concurrently, so this is one + // shared work deadline. Timed-out tasks are joined before the stage ends. + runtime_cleanup: Duration::from_secs(30), +}; + +#[cfg(test)] +const SERVICE_MANAGER_MARGIN: Duration = Duration::from_secs(20); struct ConnectionSupervisor { shutdown: watch::Sender, @@ -367,9 +383,27 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc(drain: Drain, cleanup: Cleanup) -> Result<()> +where + Drain: Future>, + Cleanup: Future>, +{ + let drain_result = drain.await; + let cleanup_result = cleanup.await; + match (drain_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(drain), Err(cleanup)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "connection drain failed: {drain}; runtime cleanup failed: {cleanup}" + ))), + } } async fn serve_connection( @@ -517,6 +551,61 @@ mod tests { ); } + #[tokio::test] + async fn cleanup_starts_after_exhausted_connection_stage() { + let mut connections = ConnectionSupervisor::new(); + let connection_active = Arc::new(AtomicBool::new(true)); + let task_active = connection_active.clone(); + let (entered_tx, entered_rx) = oneshot::channel(); + connections.spawn_task(async move { + let _active = ActiveTask(task_active); + entered_tx.send(()).expect("signal task entry"); + future::pending::<()>().await; + }); + entered_rx.await.expect("task entered"); + + let cleanup_started = Arc::new(AtomicBool::new(false)); + let cleanup_observed = cleanup_started.clone(); + let active_observed = connection_active.clone(); + let cleanup = async move { + assert!( + !active_observed.load(Ordering::Acquire), + "runtime cleanup started before connection tasks stopped" + ); + cleanup_observed.store(true, Ordering::Release); + Ok(()) + }; + let error = finish_shutdown(connections.shutdown(Duration::from_millis(10)), cleanup) + .await + .expect_err("connection timeout must be reported"); + + assert!(error.to_string().contains("connection drain timed out")); + assert!(cleanup_started.load(Ordering::Acquire)); + } + + #[test] + fn service_stop_timeout_covers_shutdown_stages() { + let unit = include_str!("../../../dist/blazed.service"); + let service_seconds = unit + .lines() + .find_map(|line| { + line.strip_prefix("TimeoutStopSec=") + .and_then(|value| value.strip_suffix('s')) + .and_then(|value| value.parse::().ok()) + }) + .expect("service stop timeout in seconds"); + let service_timeout = Duration::from_secs(service_seconds); + + assert!( + SHUTDOWN_BUDGET + .connection_drain + .saturating_add(SHUTDOWN_BUDGET.runtime_cleanup) + .saturating_add(SERVICE_MANAGER_MARGIN) + <= service_timeout, + "service manager must cover both stages and cancellation headroom" + ); + } + #[tokio::test] async fn completed_tasks_are_reaped_while_serving() { let mut connections = ConnectionSupervisor::new(); diff --git a/src/blaze/dist/blazed.service b/src/blaze/dist/blazed.service index aad769fe72..984f25e4e5 100644 --- a/src/blaze/dist/blazed.service +++ b/src/blaze/dist/blazed.service @@ -15,7 +15,7 @@ LimitNPROC=infinity LimitCORE=infinity KillMode=process KillSignal=SIGTERM -TimeoutStopSec=30 +TimeoutStopSec=80s [Install] WantedBy=multi-user.target From a12188e68648c74a8809964851a6be8a7906dd88 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 18:56:28 +0800 Subject: [PATCH 11/40] fix(blaze): bound HTTP request bodies All daemon routes previously buffered request bodies to completion. Add a validated, configurable 1 MiB default and a shared collector that rejects oversized declared lengths before polling and stops streamed bodies at the first frame above the limit. This enables every existing daemon endpoint to reject oversized input before buffering the complete body while preserving requests at or below the configured limit. Body read failures map to HTTP 400, and size violations map to HTTP 413. Use one global boundary so new routes inherit it automatically. Per-route payload limits remain outside this change. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 12 + src/blaze/README_zh.md | 11 + src/blaze/crates/blaze-core/src/config.rs | 59 +++++ src/blaze/crates/blazed/src/api.rs | 23 +- src/blaze/crates/blazed/src/error.rs | 9 +- src/blaze/crates/blazed/src/main.rs | 1 + src/blaze/crates/blazed/src/request_body.rs | 249 ++++++++++++++++++++ src/blaze/examples/config.toml | 8 + 8 files changed, 362 insertions(+), 10 deletions(-) create mode 100644 src/blaze/crates/blazed/src/request_body.rs diff --git a/src/blaze/README.md b/src/blaze/README.md index 99aaf5e6e7..a00f9f81e1 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -58,6 +58,18 @@ and a policies directory containing per-workload-class policy files. See `src/blaze/examples/` for annotated sample configurations. +### API Request Limits + +The daemon accepts request bodies up to 1 MiB by default. It checks both +declared `Content-Length` values and streamed body frames, and returns HTTP +413 when the configured limit is exceeded. Override the limit with a positive +byte count: + +```toml +[api] +max_body_bytes = 1048576 +``` + ### VM Resource Configuration Blaze resolves vCPU and memory settings using a three-layer fallback chain: diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index 63403703d9..69fc2a0bd7 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -57,6 +57,17 @@ daemon 读取 TOML 配置文件（默认：`/etc/anolisa/blaze/config.toml`） 参见 `src/blaze/examples/` 获取带注释的示例配置。 +### API 请求上限 + +daemon 默认接收不超过 1 MiB 的请求体。它会同时检查声明的 +`Content-Length` 和逐帧到达的数据；超过配置上限时返回 HTTP 413。 +可以通过正整数字节数覆盖默认值： + +```toml +[api] +max_body_bytes = 1048576 +``` + ### VM 资源配置 Blaze 使用三层回退链解析 vCPU 和内存设置： diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index fb10c93e0c..2c490167a2 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -16,6 +16,8 @@ pub struct DaemonConfig { pub daemon: DaemonSection, #[serde(default)] pub listen: ListenSection, + #[serde(default)] + pub api: ApiSection, /// Backend name → binary path mapping (e.g. `firecracker = "/usr/bin/firecracker"`). #[serde(default)] pub backends: HashMap, @@ -60,6 +62,22 @@ pub struct ListenSection { pub http_addr: String, } +/// HTTP API request limits. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ApiSection { + /// Maximum number of bytes collected from one HTTP request body. + #[serde(default = "default_max_body_bytes")] + pub max_body_bytes: usize, +} + +impl Default for ApiSection { + fn default() -> Self { + Self { + max_body_bytes: default_max_body_bytes(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PolicySection { #[serde(default = "default_policy_dir")] @@ -201,6 +219,13 @@ impl DaemonConfig { /// Validate cross-field invariants that serde cannot express. pub fn validate(&self) -> Result<()> { + if self.api.max_body_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "api.max_body_bytes must be greater than zero".to_string(), + ), + }); + } validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir) } } @@ -239,6 +264,9 @@ fn default_policy_dir() -> PathBuf { fn default_on_load_error() -> PolicyLoadErrorMode { PolicyLoadErrorMode::Fail } +fn default_max_body_bytes() -> usize { + 1024 * 1024 +} fn default_pool_warm_ttl() -> String { "30m".to_string() } @@ -285,6 +313,7 @@ mod tests { let cfg: DaemonConfig = toml::from_str("").expect("empty parses to defaults"); assert_eq!(cfg.daemon.log_level, "info"); assert_eq!(cfg.policy.on_load_error, PolicyLoadErrorMode::Fail); + assert_eq!(cfg.api.max_body_bytes, 1024 * 1024); assert!(cfg.backends.is_empty()); assert_ne!(cfg.storage.images_dir, cfg.storage.instances_dir); } @@ -311,6 +340,36 @@ mod tests { assert_eq!(cfg.backends.len(), 2); } + #[test] + fn parses_api_body_limit() { + let cfg: DaemonConfig = toml::from_str( + r#" + [api] + max_body_bytes = 4096 + "#, + ) + .expect("api config"); + assert_eq!(cfg.api.max_body_bytes, 4096); + cfg.validate().expect("positive body limit"); + } + + #[test] + fn rejects_zero_api_body_limit() { + let cfg: DaemonConfig = toml::from_str( + r#" + [api] + max_body_bytes = 0 + "#, + ) + .expect("api config"); + let error = cfg.validate().expect_err("zero body limit"); + assert!( + error + .to_string() + .contains("api.max_body_bytes must be greater than zero") + ); + } + #[test] fn rejects_equal_or_nested_storage_roots() { for (images, instances) in [ diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 307af8e1f8..3846e1ada3 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -18,7 +18,7 @@ use blaze_core::lifecycle::{BackendOwnership, SandboxInstance, SandboxState, Sta use blaze_core::policy::{ImageMetadata, RuntimeDecision, WorkloadClass, parse_duration}; use blaze_core::pool::{PoolConfig, PoolKey}; use blaze_core::storage::AcquireOpts; -use http_body_util::{BodyExt, Full}; +use http_body_util::Full; use hyper::body::{Bytes, Incoming}; use hyper::header::CONTENT_TYPE; use hyper::{Method, Request, Response, StatusCode}; @@ -29,6 +29,7 @@ use tokio::time::Instant; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; +use crate::request_body; use crate::state::ServerState; /// Top-level request handler. Always returns `Ok(Response)`; internal @@ -43,9 +44,17 @@ pub async fn handle( let path = req.uri().path().to_string(); let query = req.uri().query().unwrap_or("").to_string(); - let response = match collect_body(req).await { - Ok(body) => dispatch(&method, &path, &query, body, &state).await, - Err(e) => Err(e), + let limit = state + .config + .lock() + .map(|config| config.api.max_body_bytes) + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into())); + let response = match limit { + Ok(limit) => match request_body::collect(req, limit).await { + Ok(body) => dispatch(&method, &path, &query, body, &state).await, + Err(error) => Err(error), + }, + Err(error) => Err(error), }; let resp = match response { @@ -55,11 +64,6 @@ pub async fn handle( Ok(resp) } -async fn collect_body(req: Request) -> Result> { - let collected = req.into_body().collect().await?; - Ok(collected.to_bytes().to_vec()) -} - async fn dispatch( method: &Method, path: &str, @@ -1458,6 +1462,7 @@ mod tests { AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, }; use blaze_core::template::TemplateRegistry; + use http_body_util::BodyExt; use crate::file_provider::FileStorageProvider; use crate::spawner::{ diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index b2bd1a4f87..0bade2bd91 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -54,6 +54,12 @@ pub enum BlazeDaemonError { #[error("unsupported operation: {0}")] UnsupportedOperation(String), + #[error("request body read failed: {0}")] + RequestBody(String), + + #[error("request body too large: {actual} bytes exceeds {limit}")] + PayloadTooLarge { actual: u64, limit: usize }, + #[error("operation requires recovery: {0}")] RecoveryRequired(String), @@ -65,8 +71,9 @@ impl BlazeDaemonError { /// HTTP status code that should accompany this error in API responses. pub fn status_code(&self) -> u16 { match self { - BlazeDaemonError::BadRequest(_) => 400, + BlazeDaemonError::BadRequest(_) | BlazeDaemonError::RequestBody(_) => 400, BlazeDaemonError::NotFound(_) => 404, + BlazeDaemonError::PayloadTooLarge { .. } => 413, BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::RecoveryRequired(_) => 500, BlazeDaemonError::HttpStatus { status, .. } => *status, diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index 2760d59c19..ac9321aee4 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -15,6 +15,7 @@ mod failpoint; mod failpoint; mod file_provider; mod metrics; +mod request_body; mod spawner; mod state; diff --git a/src/blaze/crates/blazed/src/request_body.rs b/src/blaze/crates/blazed/src/request_body.rs new file mode 100644 index 0000000000..2a35ed8be1 --- /dev/null +++ b/src/blaze/crates/blazed/src/request_body.rs @@ -0,0 +1,249 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Bounded HTTP request-body collection for every daemon API route. + +use http_body_util::BodyExt; +use hyper::Request; +use hyper::body::{Body, Bytes}; +use hyper::header::CONTENT_LENGTH; + +use crate::error::{BlazeDaemonError, Result}; + +/// Collect a request body without buffering more than `limit` bytes. +pub(crate) async fn collect(req: Request, limit: usize) -> Result> +where + B: Body + Unpin, + B::Error: std::fmt::Display, +{ + if let Some(declared) = declared_body_length(&req)? + && declared > limit as u64 + { + return Err(BlazeDaemonError::PayloadTooLarge { + actual: declared, + limit, + }); + } + + let mut body = req.into_body(); + let mut collected = Vec::new(); + while let Some(frame) = body.frame().await { + let frame = frame.map_err(|error| BlazeDaemonError::RequestBody(error.to_string()))?; + if let Ok(data) = frame.into_data() { + let actual = collected.len().checked_add(data.len()).ok_or( + BlazeDaemonError::PayloadTooLarge { + actual: u64::MAX, + limit, + }, + )?; + if actual > limit { + return Err(BlazeDaemonError::PayloadTooLarge { + actual: actual as u64, + limit, + }); + } + collected.extend_from_slice(&data); + } + } + Ok(collected) +} + +fn declared_body_length(req: &Request) -> Result> { + let mut declared = None; + for value in req.headers().get_all(CONTENT_LENGTH) { + let value = value + .to_str() + .map_err(|_| BlazeDaemonError::BadRequest("invalid Content-Length".into()))?; + for item in value.split(',') { + let length = item + .trim() + .parse::() + .map_err(|_| BlazeDaemonError::BadRequest("invalid Content-Length".into()))?; + match declared { + Some(previous) if previous != length => { + return Err(BlazeDaemonError::BadRequest( + "conflicting Content-Length values".into(), + )); + } + None => declared = Some(length), + _ => {} + } + } + } + Ok(declared) +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::fmt; + use std::pin::Pin; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::task::{Context, Poll}; + + use hyper::body::Frame; + use hyper::header::{CONTENT_LENGTH, TRANSFER_ENCODING}; + + use super::*; + + #[derive(Debug)] + struct TestBodyError; + + impl fmt::Display for TestBodyError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("test body failed") + } + } + + impl std::error::Error for TestBodyError {} + + struct TestBody { + frames: VecDeque, TestBodyError>>, + polls: Arc, + panic_when_exhausted: bool, + } + + impl TestBody { + fn new( + frames: impl IntoIterator, TestBodyError>>, + polls: Arc, + ) -> Self { + Self { + frames: frames.into_iter().collect(), + polls, + panic_when_exhausted: false, + } + } + + fn panic_when_exhausted(mut self) -> Self { + self.panic_when_exhausted = true; + self + } + } + + impl Body for TestBody { + type Data = Bytes; + type Error = TestBodyError; + + fn poll_frame( + self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + let this = self.get_mut(); + this.polls.fetch_add(1, Ordering::AcqRel); + match this.frames.pop_front() { + Some(frame) => Poll::Ready(Some(frame)), + None if this.panic_when_exhausted => { + panic!("collector polled after the limit had already been exceeded") + } + None => Poll::Ready(None), + } + } + } + + #[tokio::test] + async fn accepts_body_at_declared_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"ab"))), + Ok(Frame::data(Bytes::from_static(b"cd"))), + ], + polls.clone(), + ); + let request = Request::builder() + .header(CONTENT_LENGTH, "4") + .body(body) + .expect("request"); + + assert_eq!(collect(request, 4).await.expect("body"), b"abcd"); + assert_eq!(polls.load(Ordering::Acquire), 3); + } + + #[tokio::test] + async fn rejects_large_content_length_before_polling_body() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [Ok(Frame::data(Bytes::from_static(b"body")))], + polls.clone(), + ) + .panic_when_exhausted(); + let request = Request::builder() + .header(CONTENT_LENGTH, "5") + .body(body) + .expect("request"); + + let error = collect(request, 4).await.expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(error.status_code(), 413); + assert_eq!(polls.load(Ordering::Acquire), 0); + } + + #[tokio::test] + async fn stops_collecting_chunked_body_at_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"abcd"))), + Ok(Frame::data(Bytes::from_static(b"e"))), + ], + polls.clone(), + ) + .panic_when_exhausted(); + let request = Request::builder() + .header(TRANSFER_ENCODING, "chunked") + .body(body) + .expect("request"); + + let error = collect(request, 4).await.expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(polls.load(Ordering::Acquire), 2); + } + + #[tokio::test] + async fn stops_collecting_undelimited_body_at_limit() { + let polls = Arc::new(AtomicUsize::new(0)); + let body = TestBody::new( + [ + Ok(Frame::data(Bytes::from_static(b"ab"))), + Ok(Frame::data(Bytes::from_static(b"cde"))), + ], + polls.clone(), + ) + .panic_when_exhausted(); + + let error = collect(Request::new(body), 4) + .await + .expect_err("oversized body"); + assert!(matches!( + error, + BlazeDaemonError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + assert_eq!(polls.load(Ordering::Acquire), 2); + } + + #[tokio::test] + async fn maps_body_read_failures_to_bad_request() { + let body = TestBody::new([Err(TestBodyError)], Arc::new(AtomicUsize::new(0))); + let error = collect(Request::new(body), 4) + .await + .expect_err("body read must fail"); + + assert!(matches!(error, BlazeDaemonError::RequestBody(_))); + assert_eq!(error.status_code(), 400); + } +} diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 774f1e25fc..1b1bc127d7 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -34,6 +34,14 @@ socket = "/run/blaze/api.sock" # Default: "" (disabled). http_addr = "" +# --------------------------------------------------------------------------- +# HTTP API request limits. +# --------------------------------------------------------------------------- +[api] +# Maximum request body collected by the daemon. Requests above this limit +# receive HTTP 413 before the full body is buffered. Default: 1 MiB. +max_body_bytes = 1048576 + # --------------------------------------------------------------------------- # Backend binary paths. # Maps backend name → absolute path to the executable. From 90f7737138b90a0587b0cd6f5714b02f600af173 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 18:56:46 +0800 Subject: [PATCH 12/40] fix(blaze): prevent daemon socket takeover This enables blazed to reject a competing daemon before runtime initialization or Unix socket replacement. Daemon startup now holds an exclusive lock derived from the configured API socket for the entire run. The lock file stays on disk so processes always contend on one inode. It is opened without following symbolic links and must be a single-link regular file owned by the effective user with mode 0600. Unexpected lock or endpoint state aborts startup without deleting the configured socket. An existing socket receives a bounded ownership probe. A reachable endpoint is preserved, including one created by an older daemon without the lock; only an endpoint that reports no listener is treated as stale and replaced. The guarantee is scoped to one configured API socket. Independent daemons using different sockets do not contend on this lock. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/daemon.rs | 22 +- src/blaze/crates/blazed/src/daemon_socket.rs | 543 +++++++++++++++++++ src/blaze/crates/blazed/src/error.rs | 23 + src/blaze/crates/blazed/src/main.rs | 1 + 4 files changed, 578 insertions(+), 11 deletions(-) create mode 100644 src/blaze/crates/blazed/src/daemon_socket.rs diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 3ab8c46707..c5534681d7 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -16,12 +16,13 @@ use blaze_core::template::TemplateRegistry; use hyper::server::conn::http1; use hyper::service::service_fn; use hyper_util::rt::TokioIo; -use tokio::net::{TcpListener, UnixListener}; +use tokio::net::TcpListener; use tokio::signal::unix::{SignalKind, signal}; use tokio::sync::watch; use tokio::task::{JoinError, JoinSet}; use crate::api; +use crate::daemon_socket::{DaemonLock, DaemonSocket}; use crate::error::{BlazeDaemonError, Result}; use crate::spawner::{ BubblewrapSpawner, DynSpawner, FirecrackerSpawner, MockSpawner, SpawnerRegistry, @@ -144,6 +145,8 @@ pub async fn run(config_path: &Path) -> Result<()> { tracing::info!(?config_path, "loaded daemon config"); ensure_dirs(&config)?; + let socket_path = config.daemon.socket.clone(); + let daemon_lock = DaemonLock::acquire(&socket_path)?; let policy = load_policy_engine(&config)?; let pool = PoolManager::new(); let template = TemplateRegistry::new(); @@ -177,7 +180,6 @@ pub async fn run(config_path: &Path) -> Result<()> { Arc::new(fp) }; - let socket_path = config.daemon.socket.clone(); let http_addr = config.listen.http_addr.clone(); let state = Arc::new(ServerState::build( config, @@ -190,13 +192,7 @@ pub async fn run(config_path: &Path) -> Result<()> { storage, )?); - if socket_path.exists() { - std::fs::remove_file(&socket_path)?; - } - if let Some(parent) = socket_path.parent() { - std::fs::create_dir_all(parent)?; - } - let listener = UnixListener::bind(&socket_path)?; + let listener = DaemonSocket::bind(daemon_lock).await?; tracing::info!(socket = %socket_path.display(), "blaze UDS API listening"); // Optional TCP listener for remote platform API @@ -327,7 +323,11 @@ fn load_policy_engine(cfg: &DaemonConfig) -> Result { } } -async fn serve(uds: UnixListener, tcp: Option, state: Arc) -> Result<()> { +async fn serve( + mut uds: DaemonSocket, + tcp: Option, + state: Arc, +) -> Result<()> { let mut sighup = signal(SignalKind::hangup()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGHUP handler: {e}")))?; let mut sigterm = signal(SignalKind::terminate()) @@ -381,7 +381,7 @@ async fn serve(uds: UnixListener, tcp: Option, state: Arc, + _lock: DaemonLock, +} + +impl DaemonSocket { + /// Examines and replaces the socket only under exclusive ownership. + pub(super) async fn bind(lock: DaemonLock) -> Result { + let socket_path = &lock.socket_path; + prepare_socket_path(socket_path).await?; + let listener = + UnixListener::bind(socket_path).map_err(|source| BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + })?; + Ok(Self { + listener: Some(listener), + _lock: lock, + }) + } + + /// Accepts the next client while retaining exclusive socket ownership. + pub(super) async fn accept(&self) -> io::Result<(UnixStream, tokio::net::unix::SocketAddr)> { + let listener = self.listener.as_ref().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotConnected, + "daemon socket is no longer accepting connections", + ) + })?; + listener.accept().await + } + + /// Closes the listener while retaining exclusive ownership for cleanup. + pub(super) fn stop_accepting(&mut self) { + self.listener.take(); + } +} + +/// Exclusive daemon ownership tied to one configured API socket. +pub(super) struct DaemonLock { + _file: File, + socket_path: PathBuf, +} + +impl DaemonLock { + /// Acquires ownership before daemon subsystems begin startup. + pub(super) fn acquire(socket_path: &Path) -> Result { + let lock_path = lock_path_for(socket_path); + let (file, created) = open_lock_file(&lock_path)?; + if created { + file.set_permissions(fs::Permissions::from_mode(LOCK_MODE)) + .map_err(|source| BlazeDaemonError::DaemonLockIo { + path: lock_path.clone(), + source, + })?; + } + + let opened_metadata = validate_opened_lock(&file, &lock_path).map_err(|reason| { + BlazeDaemonError::InvalidDaemonLock { + path: lock_path.clone(), + reason, + } + })?; + + // SAFETY: `file` owns a valid descriptor for the entire call. `flock` + // changes only the advisory lock associated with that open file. + let lock_result = unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if lock_result != 0 { + let source = io::Error::last_os_error(); + if source.kind() == io::ErrorKind::WouldBlock { + return Err(BlazeDaemonError::DaemonAlreadyRunning { + socket: socket_path.to_path_buf(), + }); + } + return Err(BlazeDaemonError::DaemonLockIo { + path: lock_path, + source, + }); + } + + validate_locked_path(&lock_path, &opened_metadata).map_err(|reason| { + BlazeDaemonError::InvalidDaemonLock { + path: lock_path, + reason, + } + })?; + + // The file is deliberately left on disk. Removing an advisory-lock + // file would let a new process lock a different inode during teardown. + Ok(Self { + _file: file, + socket_path: socket_path.to_path_buf(), + }) + } +} + +impl Drop for DaemonLock { + fn drop(&mut self) { + // SAFETY: `_file` remains open while its advisory lock is released. + // Process termination still closes the descriptor and releases the + // lock if this destructor cannot run. + unsafe { + libc::flock(self._file.as_raw_fd(), libc::LOCK_UN); + } + } +} + +fn lock_path_for(socket_path: &Path) -> PathBuf { + let mut lock_path = OsString::from(socket_path.as_os_str()); + lock_path.push(".lock"); + PathBuf::from(lock_path) +} + +fn open_lock_file(path: &Path) -> Result<(File, bool)> { + match lock_options(true).open(path) { + Ok(file) => Ok((file, true)), + Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { + let file = lock_options(false).open(path).map_err(|source| { + BlazeDaemonError::DaemonLockIo { + path: path.to_path_buf(), + source, + } + })?; + Ok((file, false)) + } + Err(source) => Err(BlazeDaemonError::DaemonLockIo { + path: path.to_path_buf(), + source, + }), + } +} + +fn lock_options(create_new: bool) -> OpenOptions { + let mut options = OpenOptions::new(); + options + .read(true) + .write(true) + .create_new(create_new) + .mode(LOCK_MODE) + .custom_flags(libc::O_CLOEXEC | libc::O_NOFOLLOW); + options +} + +fn validate_opened_lock(file: &File, path: &Path) -> std::result::Result { + let metadata = file + .metadata() + .map_err(|error| format!("cannot inspect opened file: {error}"))?; + if !metadata.file_type().is_file() { + return Err("lock target is not a regular file".to_string()); + } + if metadata.mode() & 0o7777 != LOCK_MODE { + return Err(format!( + "mode must be {LOCK_MODE:#o}, found {:#o}", + metadata.mode() & 0o7777 + )); + } + // SAFETY: `geteuid` has no preconditions and does not modify memory. + let effective_uid = unsafe { libc::geteuid() }; + if metadata.uid() != effective_uid { + return Err(format!( + "owner uid {} does not match effective uid {effective_uid}", + metadata.uid() + )); + } + if metadata.nlink() != 1 { + return Err(format!( + "lock file must have one link, found {}", + metadata.nlink() + )); + } + + let path_metadata = + fs::symlink_metadata(path).map_err(|error| format!("cannot inspect path: {error}"))?; + if path_metadata.file_type().is_symlink() { + return Err("lock path is a symbolic link".to_string()); + } + if path_metadata.dev() != metadata.dev() || path_metadata.ino() != metadata.ino() { + return Err("lock path changed while it was opened".to_string()); + } + Ok(metadata) +} + +fn validate_locked_path( + path: &Path, + opened_metadata: &fs::Metadata, +) -> std::result::Result<(), String> { + let path_metadata = fs::symlink_metadata(path) + .map_err(|error| format!("cannot inspect locked path: {error}"))?; + if path_metadata.file_type().is_symlink() { + return Err("lock path became a symbolic link".to_string()); + } + if path_metadata.dev() != opened_metadata.dev() || path_metadata.ino() != opened_metadata.ino() + { + return Err("lock path changed while ownership was acquired".to_string()); + } + if path_metadata.mode() & 0o7777 != LOCK_MODE { + return Err(format!( + "mode changed while ownership was acquired: found {:#o}", + path_metadata.mode() & 0o7777 + )); + } + Ok(()) +} + +async fn prepare_socket_path(socket_path: &Path) -> Result<()> { + match fs::symlink_metadata(socket_path) { + Ok(metadata) if metadata.file_type().is_socket() => { + match timeout(SOCKET_PROBE_TIMEOUT, UnixStream::connect(socket_path)).await { + Ok(Ok(_stream)) => Err(BlazeDaemonError::DaemonAlreadyRunning { + socket: socket_path.to_path_buf(), + }), + Ok(Err(source)) if source.kind() == io::ErrorKind::ConnectionRefused => { + fs::remove_file(socket_path).map_err(|source| { + BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + } + }) + } + Ok(Err(source)) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Ok(Err(source)) => Err(BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + }), + Err(_) => Err(BlazeDaemonError::InvalidDaemonSocket { + path: socket_path.to_path_buf(), + reason: "existing socket did not complete the ownership probe".to_string(), + }), + } + } + Ok(metadata) => { + let kind = if metadata.file_type().is_symlink() { + "symbolic link" + } else if metadata.is_dir() { + "directory" + } else { + "non-socket file" + }; + Err(BlazeDaemonError::InvalidDaemonSocket { + path: socket_path.to_path_buf(), + reason: format!("existing path is a {kind}"), + }) + } + Err(source) if source.kind() == io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(BlazeDaemonError::DaemonSocketIo { + path: socket_path.to_path_buf(), + source, + }), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::symlink; + use std::os::unix::net::UnixListener as StdUnixListener; + use std::process::{Command, Stdio}; + + const ABRUPT_EXIT_SOCKET_ENV: &str = "BLAZE_TEST_ABRUPT_EXIT_SOCKET"; + const ABRUPT_EXIT_READY_ENV: &str = "BLAZE_TEST_ABRUPT_EXIT_READY"; + static SOCKET_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + + async fn serialize_socket_test() -> tokio::sync::MutexGuard<'static, ()> { + // The abrupt-exit test starts a helper process. Between fork and exec, + // that helper can inherit another concurrent test's listener and keep + // its endpoint reachable after the parent test drops the listener. + SOCKET_TEST_LOCK.lock().await + } + + fn socket_inode(path: &Path) -> u64 { + fs::symlink_metadata(path).expect("socket metadata").ino() + } + + async fn claim_and_bind(path: &Path) -> Result { + DaemonSocket::bind(DaemonLock::acquire(path)?).await + } + + async fn claim_and_bind_after_listener_handoff(path: &Path) -> Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(1); + loop { + // Lock release must be immediate. Only bind is retried because an + // unrelated child between fork and exec can briefly inherit the + // old CLOEXEC listener and keep its endpoint reachable. + let lock = DaemonLock::acquire(path)?; + match DaemonSocket::bind(lock).await { + Err(BlazeDaemonError::DaemonAlreadyRunning { .. }) + if tokio::time::Instant::now() < deadline => + { + tokio::time::sleep(Duration::from_millis(10)).await; + } + result => return result, + } + } + } + + #[tokio::test] + async fn second_daemon_cannot_replace_owned_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let first = claim_and_bind(&socket_path) + .await + .expect("first daemon binds"); + let first_inode = socket_inode(&socket_path); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("second daemon must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), first_inode); + drop(first); + } + + #[tokio::test] + async fn released_lock_can_be_reacquired_and_stale_socket_replaced() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let first = claim_and_bind(&socket_path) + .await + .expect("first daemon binds"); + drop(first); + + let second = claim_and_bind_after_listener_handoff(&socket_path) + .await + .expect("released lock is reusable"); + + assert!( + fs::symlink_metadata(&socket_path) + .expect("replacement socket metadata") + .file_type() + .is_socket() + ); + drop(second); + } + + #[tokio::test] + async fn closing_listener_retains_daemon_ownership() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let mut daemon = claim_and_bind(&socket_path).await.expect("daemon binds"); + + daemon.stop_accepting(); + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("closed listener must retain daemon ownership"); + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + + drop(daemon); + let recovered = DaemonLock::acquire(&socket_path).expect("ownership releases with daemon"); + drop(recovered); + } + + #[tokio::test] + async fn lock_is_released_after_owner_process_exits_abruptly() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let ready_path = temp.path().join("lock-ready"); + let status = Command::new(std::env::current_exe().expect("current test binary")) + .arg("--exact") + .arg("daemon_socket::tests::abrupt_exit_lock_helper") + .arg("--nocapture") + .env(ABRUPT_EXIT_SOCKET_ENV, &socket_path) + .env(ABRUPT_EXIT_READY_ENV, &ready_path) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("run abrupt-exit helper"); + + assert_eq!(status.code(), Some(73)); + assert_eq!( + fs::read(&ready_path).expect("helper acquired lock"), + b"locked" + ); + + let recovered = DaemonLock::acquire(&socket_path).expect("kernel released process lock"); + drop(recovered); + } + + #[test] + fn abrupt_exit_lock_helper() { + let Some(socket_path) = std::env::var_os(ABRUPT_EXIT_SOCKET_ENV) else { + return; + }; + let Some(ready_path) = std::env::var_os(ABRUPT_EXIT_READY_ENV) else { + return; + }; + let _lock = + DaemonLock::acquire(Path::new(&socket_path)).expect("helper acquires daemon lock"); + fs::write(ready_path, b"locked").expect("publish helper readiness"); + + // SAFETY: `_exit` terminates only this dedicated helper process. It + // intentionally skips Rust destructors to exercise kernel lock release. + unsafe { + libc::_exit(73); + } + } + + #[tokio::test] + async fn stale_socket_is_untouched_when_lock_is_held() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let lock = DaemonLock::acquire(&socket_path).expect("hold daemon lock"); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("competing daemon must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), stale_inode); + drop(lock); + + let daemon = claim_and_bind_after_listener_handoff(&socket_path) + .await + .expect("owner may replace stale socket"); + assert!( + fs::symlink_metadata(&socket_path) + .expect("replacement socket metadata") + .file_type() + .is_socket() + ); + drop(daemon); + } + + #[tokio::test] + async fn symlinked_lock_is_rejected_without_touching_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let target = temp.path().join("lock-target"); + File::create(&target).expect("create target"); + symlink(&target, lock_path_for(&socket_path)).expect("create lock symlink"); + + assert!(DaemonLock::acquire(&socket_path).is_err()); + assert_eq!(socket_inode(&socket_path), stale_inode); + } + + #[tokio::test] + async fn live_socket_without_lock_is_preserved() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let legacy_listener = + StdUnixListener::bind(&socket_path).expect("bind legacy daemon socket"); + let original_inode = socket_inode(&socket_path); + let lock = DaemonLock::acquire(&socket_path).expect("acquire new daemon lock"); + + let error = DaemonSocket::bind(lock) + .await + .err() + .expect("live socket must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::DaemonAlreadyRunning { .. } + )); + assert_eq!(socket_inode(&socket_path), original_inode); + drop(legacy_listener); + } + + #[tokio::test] + async fn insecure_lock_mode_is_rejected_without_touching_socket() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + let stale = StdUnixListener::bind(&socket_path).expect("bind stale socket"); + drop(stale); + let stale_inode = socket_inode(&socket_path); + let lock_path = lock_path_for(&socket_path); + let lock = File::create(&lock_path).expect("create lock"); + lock.set_permissions(fs::Permissions::from_mode(0o644)) + .expect("set insecure mode"); + + let error = DaemonLock::acquire(&socket_path) + .err() + .expect("insecure lock must be rejected"); + + assert!(matches!(error, BlazeDaemonError::InvalidDaemonLock { .. })); + assert_eq!(socket_inode(&socket_path), stale_inode); + } + + #[tokio::test] + async fn non_socket_endpoint_is_rejected_without_removal() { + let _test_guard = serialize_socket_test().await; + let temp = tempfile::tempdir().expect("tempdir"); + let socket_path = temp.path().join("api.sock"); + fs::write(&socket_path, b"do not remove").expect("write endpoint sentinel"); + + let lock = DaemonLock::acquire(&socket_path).expect("acquire daemon lock"); + let error = DaemonSocket::bind(lock) + .await + .err() + .expect("non-socket endpoint must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::InvalidDaemonSocket { .. } + )); + assert_eq!( + fs::read(&socket_path).expect("sentinel remains"), + b"do not remove" + ); + } +} diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index 0bade2bd91..d3c9b60e20 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -63,6 +63,29 @@ pub enum BlazeDaemonError { #[error("operation requires recovery: {0}")] RecoveryRequired(String), + #[error("another blaze daemon already owns API socket {socket}")] + DaemonAlreadyRunning { socket: PathBuf }, + + #[error("cannot access daemon lock {path}: {source}")] + DaemonLockIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + #[error("refusing daemon lock {path}: {reason}")] + InvalidDaemonLock { path: PathBuf, reason: String }, + + #[error("cannot prepare daemon API socket {path}: {source}")] + DaemonSocketIo { + path: PathBuf, + #[source] + source: std::io::Error, + }, + + #[error("refusing daemon API socket {path}: {reason}")] + InvalidDaemonSocket { path: PathBuf, reason: String }, + #[error("internal error: {0}")] Internal(String), } diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index ac9321aee4..4f654c85a9 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -7,6 +7,7 @@ mod api; mod cli; mod daemon; +mod daemon_socket; mod error; #[cfg(feature = "test-failpoints")] mod failpoint; From 1607ee1849738b2cbe98b7a1291a30be3a3a6f7d Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 03:00:02 +0800 Subject: [PATCH 13/40] fix(blaze): publish lifecycle owners atomically Prepare a new lifecycle record in a non-owner staging directory and sync its file and directory before publishing the UUID owner directory. The previous sequence exposed the final UUID directory before state.json existed and did not sync the state root. A crash could therefore leave an owner that startup could neither load nor classify. Rename the complete staging directory into place and sync the state root. Existing owner updates keep their atomic file replacement and now sync both the file and owner directory. This lets startup observe either no owner or a complete lifecycle record. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/lifecycle.rs | 154 ++++++++++++++++--- 1 file changed, 136 insertions(+), 18 deletions(-) diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index 4fdfac9698..e17e71f81e 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 //! Sandbox lifecycle state machine + JSON persistence. -use std::fs; +use std::fs::{self, File}; +use std::io::Write; use std::path::{Path, PathBuf}; use chrono::{DateTime, Utc}; @@ -150,7 +151,10 @@ impl SandboxInstance { /// rename via `state.json.tmp` to avoid torn reads on daemon restart. pub fn persist(&self, state_dir: &Path) -> Result<()> { self.persist_with(state_dir, |tmp_path, final_path, json| { - fs::write(tmp_path, json)?; + let mut file = File::create(tmp_path)?; + file.write_all(json)?; + file.sync_all()?; + drop(file); fs::rename(tmp_path, final_path)?; Ok(()) }) @@ -160,26 +164,67 @@ impl SandboxInstance { where F: FnOnce(&Path, &Path, &[u8]) -> Result<()>, { - let dir = state_dir.join(self.id.to_string()); + self.persist_with_directory_sync(state_dir, publish, |directory| { + File::open(directory)?.sync_all()?; + Ok(()) + }) + } + + fn persist_with_directory_sync( + &self, + state_dir: &Path, + publish: F, + mut sync_directory: S, + ) -> Result<()> + where + F: FnOnce(&Path, &Path, &[u8]) -> Result<()>, + S: FnMut(&Path) -> Result<()>, + { + let owner_dir = state_dir.join(self.id.to_string()); let json = serde_json::to_vec_pretty(self)?; fs::create_dir_all(state_dir)?; - let created_owner_dir = match fs::create_dir(&dir) { - Ok(()) => true, - Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => false, - Err(error) => return Err(error.into()), - }; - let final_path = dir.join("state.json"); - let tmp_path = dir.join("state.json.tmp"); - let result = publish(&tmp_path, &final_path, &json); - if result.is_err() { - let _ = fs::remove_file(&tmp_path); - if created_owner_dir { - // Remove only an empty directory. If another owned artifact - // appeared, retain the directory so startup fails closed. - let _ = fs::remove_dir(&dir); + + match fs::symlink_metadata(&owner_dir) { + Ok(metadata) if metadata.file_type().is_dir() => { + let final_path = owner_dir.join("state.json"); + let tmp_path = owner_dir.join("state.json.tmp"); + let result = publish(&tmp_path, &final_path, &json); + if result.is_err() { + let _ = fs::remove_file(&tmp_path); + } + result?; + sync_directory(&owner_dir) } + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!( + "lifecycle owner {} is not a real directory", + owner_dir.display() + ), + ) + .into()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + let staging_dir = + state_dir.join(format!(".state-{}-{}.tmp", self.id, Uuid::new_v4())); + fs::create_dir(&staging_dir)?; + let final_path = staging_dir.join("state.json"); + let tmp_path = staging_dir.join("state.json.tmp"); + if let Err(error) = publish(&tmp_path, &final_path, &json) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error); + } + if let Err(error) = sync_directory(&staging_dir) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error); + } + if let Err(error) = fs::rename(&staging_dir, &owner_dir) { + let _ = fs::remove_dir_all(&staging_dir); + return Err(error.into()); + } + sync_directory(state_dir) + } + Err(error) => Err(error.into()), } - result } /// Reload an instance previously persisted via [`Self::persist`]. @@ -212,6 +257,8 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { #[cfg(test)] mod tests { + use std::cell::Cell; + use super::*; fn fresh() -> SandboxInstance { @@ -315,6 +362,77 @@ mod tests { assert!(error.to_string().contains("injected publication failure")); assert!(!tmp.path().join(instance.id.to_string()).exists()); + assert_eq!( + std::fs::read_dir(tmp.path()) + .expect("state directory") + .count(), + 0 + ); + } + + #[test] + fn first_persist_syncs_the_staged_owner_and_state_root() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let sync_count = Cell::new(0); + + instance + .persist_with_directory_sync( + tmp.path(), + |tmp_path, final_path, json| { + std::fs::write(tmp_path, json)?; + std::fs::rename(tmp_path, final_path)?; + Ok(()) + }, + |_| { + sync_count.set(sync_count.get() + 1); + Ok(()) + }, + ) + .expect("publish lifecycle"); + + assert_eq!(sync_count.get(), 2); + assert_eq!( + SandboxInstance::load(tmp.path(), instance.id) + .expect("published lifecycle") + .id, + instance.id + ); + } + + #[test] + fn parent_sync_failure_preserves_the_published_owner() { + let tmp = tempfile::tempdir().expect("tmp"); + let instance = fresh(); + let sync_count = Cell::new(0); + + let error = instance + .persist_with_directory_sync( + tmp.path(), + |tmp_path, final_path, json| { + std::fs::write(tmp_path, json)?; + std::fs::rename(tmp_path, final_path)?; + Ok(()) + }, + |_| { + let next = sync_count.get() + 1; + sync_count.set(next); + if next == 2 { + Err(std::io::Error::other("injected parent sync failure").into()) + } else { + Ok(()) + } + }, + ) + .expect_err("parent sync result is uncertain"); + + assert!(error.to_string().contains("injected parent sync failure")); + assert_eq!( + SandboxInstance::load(tmp.path(), instance.id) + .expect("published lifecycle remains") + .id, + instance.id + ); } #[test] From 3cc15616edeba0a5ad8de64f14cb56c38522fc8d Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 03:04:02 +0800 Subject: [PATCH 14/40] fix(blaze): reject linked storage artifacts Reconstruct provider-owned slots only when the slot root and every required artifact are direct filesystem objects. The previous metadata checks followed links, so a restored sandbox could silently read or write files outside the provider-owned slot after local state changed. Use link-aware metadata and classify linked roots or artifacts as incomplete. This keeps restart and dirty flush bound to the provider's own directory tree. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/file_provider.rs | 156 ++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index 0c9f096f16..fb78fdb67b 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -86,7 +86,7 @@ async fn require_slot_path( path: &Path, required_type: RequiredPathType, ) -> Result<()> { - match tokio::fs::metadata(path).await { + match tokio::fs::symlink_metadata(path).await { Ok(metadata) if required_type.matches(&metadata) => Ok(()), Ok(_) => Err(BlazeError::StorageIncomplete { instance_id: instance_id.to_string(), @@ -234,12 +234,19 @@ impl StorageProvider for FileStorageProvider { // Never trust paths carried by a runtime or persisted slot. Rebuild // the complete provider-owned artifact set from the validated ID. let canonical = self.slot_for_id(&slot.id)?; + require_slot_path( + &slot.id, + &canonical.instance_dir, + RequiredPathType::Directory, + ) + .await?; for path in [ &canonical.rootfs_path, &canonical.mem_path, &canonical.mem_diff_path, &canonical.rootfs_diff_path, ] { + require_slot_path(&slot.id, path, RequiredPathType::File).await?; let file = tokio::fs::OpenOptions::new() .read(true) .write(true) @@ -521,6 +528,153 @@ mod tests { )); } + #[cfg(unix)] + #[tokio::test] + async fn reconstruct_rejects_a_linked_slot_root() { + use std::os::unix::fs::symlink; + + let storage = tempfile::TempDir::new().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + for artifact in ["rootfs.ext4", "mem.bin", "mem.diff", "rootfs.diff"] { + tokio::fs::write(target.path().join(artifact), b"external") + .await + .unwrap(); + } + symlink(target.path(), storage.path().join("linked-slot")).unwrap(); + let provider = FileStorageProvider::new(storage.path().to_path_buf()); + + let error = provider + .reconstruct("linked-slot") + .await + .expect_err("linked slot root must be rejected"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "directory", + } if instance_id == "linked-slot" && path == &storage.path().join("linked-slot") + )); + assert!( + std::fs::symlink_metadata(storage.path().join("linked-slot")) + .unwrap() + .file_type() + .is_symlink() + ); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconstruct_rejects_a_linked_slot_artifact() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(temp.path().to_path_buf()); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "linked-artifact".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + tokio::fs::remove_file(&slot.mem_diff_path).await.unwrap(); + let external = temp.path().join("external-memory-diff"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.mem_diff_path).unwrap(); + + let error = provider + .reconstruct("linked-artifact") + .await + .expect_err("linked artifact must be rejected"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "file", + } if instance_id == "linked-artifact" && path == &slot.mem_diff_path + )); + assert!( + std::fs::symlink_metadata(&slot.mem_diff_path) + .unwrap() + .file_type() + .is_symlink() + ); + assert!(external.is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn flush_rejects_a_linked_slot_root() { + use std::os::unix::fs::symlink; + + let storage = tempfile::TempDir::new().unwrap(); + let target = tempfile::TempDir::new().unwrap(); + for artifact in ["rootfs.ext4", "mem.bin", "mem.diff", "rootfs.diff"] { + tokio::fs::write(target.path().join(artifact), b"external") + .await + .unwrap(); + } + symlink(target.path(), storage.path().join("linked-flush")).unwrap(); + let provider = FileStorageProvider::new(storage.path().to_path_buf()); + let slot = provider.slot_for_id("linked-flush").unwrap(); + + let error = provider + .flush_dirty(&slot) + .await + .expect_err("linked slot root must not be flushed"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "directory", + } if instance_id == "linked-flush" && path == &storage.path().join("linked-flush") + )); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn flush_rejects_a_linked_slot_artifact() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(temp.path().to_path_buf()); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "linked-flush-artifact".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + tokio::fs::remove_file(&slot.mem_diff_path).await.unwrap(); + let external = temp.path().join("external-memory-diff"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.mem_diff_path).unwrap(); + + let error = provider + .flush_dirty(&slot) + .await + .expect_err("linked artifact must not be flushed"); + + assert!(matches!( + error, + BlazeError::StorageIncomplete { + ref instance_id, + ref path, + expected: "file", + } if instance_id == "linked-flush-artifact" && path == &slot.mem_diff_path + )); + assert_eq!(tokio::fs::read(&external).await.unwrap(), b"external"); + } + #[tokio::test] async fn flush_rederives_canonical_paths_from_slot_id() { let temp = tempfile::TempDir::new().unwrap(); From ad7191b8275481401bb72958930fae4b9fbe5cae Mon Sep 17 00:00:00 2001 From: Weisson Date: Wed, 29 Jul 2026 17:04:29 +0800 Subject: [PATCH 15/40] feat(blaze): journal sandbox ownership changes Sandbox metadata previously recorded only lifecycle state. A crash between provider allocation, backend start, and final publication could not distinguish completed work from an operation that might still own resources. Add a durable create or destroy journal and the RecoveryRequired state. State publication now syncs the temporary file, atomically renames it, and syncs the owner directory. A failed first publication removes only the empty directory it created, and a new operation cannot overwrite an unfinished journal. This enables later lifecycle management to discover interrupted work after a restart without mistaking it for a clean sandbox. Older records remain readable through conservative ownership defaults. This commit defines and persists the model only; it does not start or release runtime resources. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/error.rs | 4 + src/blaze/crates/blaze-core/src/lifecycle.rs | 167 ++++++++++++++++++- 2 files changed, 170 insertions(+), 1 deletion(-) diff --git a/src/blaze/crates/blaze-core/src/error.rs b/src/blaze/crates/blaze-core/src/error.rs index 8c2f34c372..9f0db3c7f4 100644 --- a/src/blaze/crates/blaze-core/src/error.rs +++ b/src/blaze/crates/blaze-core/src/error.rs @@ -29,6 +29,10 @@ pub enum BlazeError { #[error("invalid sandbox state transition: {from} -> {to}")] InvalidStateTransition { from: String, to: String }, + /// A lifecycle caller tried to replace an unfinished durable operation. + #[error("sandbox operation already in progress: active={active}, requested={requested}")] + OperationInProgress { active: String, requested: String }, + #[error("template registry error: {msg}")] TemplateError { msg: String }, diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index e17e71f81e..c270e546d8 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -22,6 +22,7 @@ pub enum SandboxState { Running, Paused, Checkpointed, + RecoveryRequired, Reset, Warm, Destroyed, @@ -35,6 +36,7 @@ impl SandboxState { SandboxState::Running => "running", SandboxState::Paused => "paused", SandboxState::Checkpointed => "checkpointed", + SandboxState::RecoveryRequired => "recovery-required", SandboxState::Reset => "reset", SandboxState::Warm => "warm", SandboxState::Destroyed => "destroyed", @@ -42,6 +44,40 @@ impl SandboxState { } } +/// Persisted multi-step operation used for crash diagnosis and recovery. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperationKind { + /// Sandbox creation is acquiring resources or starting a backend. + Create, + /// Runtime resources are being destroyed. + Destroy, +} + +impl OperationKind { + pub const fn as_str(&self) -> &'static str { + match self { + OperationKind::Create => "create", + OperationKind::Destroy => "destroy", + } + } +} + +impl std::fmt::Display for OperationKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Durable journal entry for one active lifecycle operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OperationJournal { + /// Operation being performed. + pub kind: OperationKind, + /// UTC time at which the operation became externally visible. + pub started_at: DateTime, +} + impl std::fmt::Display for SandboxState { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.as_str()) @@ -86,6 +122,9 @@ pub struct SandboxInstance { /// Last durably known backend ownership state. #[serde(default)] pub backend_ownership: BackendOwnership, + /// Active multi-step operation, if any. + #[serde(default)] + pub operation: Option, } impl SandboxInstance { @@ -111,7 +150,47 @@ impl SandboxInstance { updated_at: now, policy_name, backend_ownership: BackendOwnership::NotStarted, + operation: None, + } + } + + /// Record a new operation before starting its first owned-resource + /// mutation. An unfinished journal must be recovered rather than silently + /// replaced by a later request. + pub fn begin_operation(&mut self, kind: OperationKind) -> Result<()> { + if let Some(active) = &self.operation { + return Err(BlazeError::OperationInProgress { + active: active.kind.to_string(), + requested: kind.to_string(), + }); + } + self.operation = Some(OperationJournal { + kind, + started_at: Utc::now(), + }); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Transfer an interrupted lifecycle operation to destroy recovery. + /// + /// Cleanup is the only operation allowed to supersede an unfinished + /// journal because it releases, rather than acquires, owned resources. + pub fn begin_destroy_recovery(&mut self) { + if self.operation.as_ref().map(|operation| operation.kind) == Some(OperationKind::Destroy) { + return; } + self.operation = Some(OperationJournal { + kind: OperationKind::Destroy, + started_at: Utc::now(), + }); + self.updated_at = Utc::now(); + } + + /// Clear the marker before atomically persisting the final state. + pub fn finish_operation(&mut self) { + self.operation = None; + self.updated_at = Utc::now(); } /// Apply a state transition. Returns @@ -153,6 +232,7 @@ impl SandboxInstance { self.persist_with(state_dir, |tmp_path, final_path, json| { let mut file = File::create(tmp_path)?; file.write_all(json)?; + file.write_all(b"\n")?; file.sync_all()?; drop(file); fs::rename(tmp_path, final_path)?; @@ -237,11 +317,16 @@ impl SandboxInstance { } fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { - use SandboxState::{Checkpointed, Creating, Destroyed, Paused, Pending, Reset, Running, Warm}; + use SandboxState::{ + Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Reset, Running, Warm, + }; if to == Destroyed { // `* → destroyed` is always valid (terminal sink). return from != Destroyed; } + if to == RecoveryRequired { + return !matches!(from, Destroyed | RecoveryRequired); + } match (from, to) { (Pending, Creating) => true, (Creating, Running) => true, @@ -309,6 +394,29 @@ mod tests { )); } + #[test] + fn recovery_required_can_finish_but_cannot_be_reentered() { + let mut inst = fresh(); + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.transition(SandboxState::RecoveryRequired) + .expect("recovery required"); + + let repeated = inst.transition(SandboxState::RecoveryRequired); + assert!(matches!( + repeated, + Err(BlazeError::InvalidStateTransition { .. }) + )); + + inst.transition(SandboxState::Destroyed) + .expect("destroyed from recovery"); + let terminal = inst.transition(SandboxState::RecoveryRequired); + assert!(matches!( + terminal, + Err(BlazeError::InvalidStateTransition { .. }) + )); + } + #[test] fn illegal_pending_to_running() { let mut inst = fresh(); @@ -451,4 +559,61 @@ mod tests { assert!(owner_dir.join("backend.pid").exists()); } + + #[test] + fn legacy_state_without_optional_fields_deserializes() { + let inst = fresh(); + let value = serde_json::json!({ + "id": inst.id, + "state": "running", + "backend": "mock", + "workload_class": "agent-rl", + "image_digest": "sha256:old", + "start_path": "cold", + "created_at": inst.created_at, + "updated_at": inst.updated_at, + "policy_name": "legacy" + }); + let loaded: SandboxInstance = serde_json::from_value(value).expect("legacy state"); + assert!(loaded.operation.is_none()); + assert_eq!(loaded.backend_ownership, BackendOwnership::Unknown); + } + + #[test] + fn create_journal_round_trips() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + instance.persist(tmp.path()).expect("persist"); + + let mut loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + assert_eq!( + loaded.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); + loaded.finish_operation(); + assert!(loaded.operation.is_none()); + } + + #[test] + fn unfinished_journal_cannot_be_overwritten() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone().expect("journal"); + + let error = instance + .begin_operation(OperationKind::Destroy) + .expect_err("unfinished operation must be preserved"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "destroy" + )); + assert_eq!(instance.operation, Some(journal)); + } } From 52e927fccc87dc57b3656e79a6468f28e88b1317 Mon Sep 17 00:00:00 2001 From: Weisson Date: Wed, 29 Jul 2026 18:15:54 +0800 Subject: [PATCH 16/40] feat(blaze): manage recoverable sandboxes Instance handlers previously coordinated metadata, provider slots, backend processes, and warm claims themselves. Startup and shutdown had no single owner that could enumerate incomplete work and retry every retained resource. Add SandboxManager as the owner of create, warm activation, destroy, startup reconciliation, and graceful shutdown cleanup. Every mutation uses one per-sandbox operation lock, persists intent before changing resources, and retains incomplete ownership as RecoveryRequired. This enables Blaze to create, list, inspect, and destroy sandboxes while keeping partial failures recoverable. Startup reconciliation bounds each record and continues after a failure. Graceful shutdown starts every owned sandbox concurrently under one shared deadline, cancels and joins tasks that overrun it, and reports all remaining owners. Reserved checkpoint and reset actions use the same quiescent-state gate. They still return HTTP 501 without changing runtime or persisted state until their backend and storage operations are implemented. This commit does not add sandbox-named routes, guest commands, VM networking, checkpoint capture, hibernation, or background retries. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/Cargo.toml | 1 + src/blaze/crates/blazed/src/api.rs | 2100 ++++++++++------- src/blaze/crates/blazed/src/daemon.rs | 14 + src/blaze/crates/blazed/src/error.rs | 5 + src/blaze/crates/blazed/src/main.rs | 1 + src/blaze/crates/blazed/src/sandbox.rs | 6 + .../crates/blazed/src/sandbox/manager.rs | 1204 ++++++++++ src/blaze/crates/blazed/src/state.rs | 100 +- 8 files changed, 2465 insertions(+), 966 deletions(-) create mode 100644 src/blaze/crates/blazed/src/sandbox.rs create mode 100644 src/blaze/crates/blazed/src/sandbox/manager.rs diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index 936f81f896..2082221b5d 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -38,3 +38,4 @@ libc = { workspace = true } [dev-dependencies] tempfile = "3" +tokio = { workspace = true, features = ["test-util"] } diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 3846e1ada3..e753ab5cd8 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -11,13 +11,11 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; -use blaze_core::BlazeError; -use blaze_core::backend::{BackendKind, BackendStatus, SpawnRequest, select_backend}; +use blaze_core::backend::{BackendKind, BackendStatus, select_backend}; use blaze_core::kernel::HookKind; -use blaze_core::lifecycle::{BackendOwnership, SandboxInstance, SandboxState, StartPath}; +use blaze_core::lifecycle::{SandboxInstance, SandboxState, StartPath}; use blaze_core::policy::{ImageMetadata, RuntimeDecision, WorkloadClass, parse_duration}; use blaze_core::pool::{PoolConfig, PoolKey}; -use blaze_core::storage::AcquireOpts; use http_body_util::Full; use hyper::body::{Bytes, Incoming}; use hyper::header::CONTENT_TYPE; @@ -30,6 +28,7 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::request_body; +use crate::sandbox::CreateSandbox; use crate::state::ServerState; /// Top-level request handler. Always returns `Ok(Response)`; internal @@ -168,55 +167,39 @@ struct CreateInstanceResp { } fn list_instances(state: &Arc) -> Result>> { - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - let list: Vec<&SandboxInstance> = map.values().collect(); - json_ok(&list) + json_ok(&state.manager.list()?) } fn get_instance(state: &Arc, id: &str) -> Result>> { - let uuid = parse_uuid(id)?; - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - let inst = map - .get(&uuid) - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; - json_ok(inst) + json_ok(&state.manager.get(parse_uuid(id)?)?) } async fn create_instance(state: &Arc, body: &[u8]) -> Result>> { let req: CreateInstanceReq = serde_json::from_slice(body) .map_err(|e| BlazeDaemonError::BadRequest(format!("invalid create body: {e}")))?; - let img = ImageMetadata { + let image = ImageMetadata { digest: req.image_digest.clone(), workload_class: Some(req.workload_class), kernel_version: req.kernel_version.clone(), }; - - // 1. Policy evaluation. let decision = { let engine = state .policy .lock() .map_err(|_| BlazeDaemonError::Internal("policy lock poisoned".into()))?; - match engine.evaluate(&req.labels, &img) { - Ok(d) => d, - Err(e) => { + match engine.evaluate(&req.labels, &image) { + Ok(decision) => decision, + Err(error) => { state.metrics.inc(&state.metrics.policy_eval_failures); - return Err(e.into()); + return Err(error.into()); } } }; - // 2. Backend selection. Constrain availability to the daemon's active - // spawner — only the backend that was actually probed at boot can execute. + // Constrain availability to the implementation selected at daemon boot. let availability: Vec = { - let cfg = state + let config = state .config .lock() .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))?; @@ -226,10 +209,10 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result, body: &[u8]) -> Result b, - Err(e) => { - if state.active_backend == BackendKind::Mock { - *decision.backend_priority.first().ok_or_else(|| { - BlazeDaemonError::Internal("policy has empty backend_priority".into()) - })? - } else { - return Err(e.into()); - } + Ok(backend) => backend, + Err(_) if state.active_backend == BackendKind::Mock => { + *decision.backend_priority.first().ok_or_else(|| { + BlazeDaemonError::Internal("policy has empty backend_priority".into()) + })? } + Err(error) => return Err(error.into()), }; - // Policy chooses an allowed backend preference. Runtime ownership must - // record the spawner that actually serves the request. In portable Mock - // mode those can differ because Mock is an explicit local fallback. let runtime_backend = if state.active_backend == BackendKind::Mock { BackendKind::Mock } else { policy_backend }; - - let pool_key = PoolKey::new( - runtime_backend, - decision.workload_class, - req.image_digest.clone(), - ); - if decision.pool_eligible { - if let Some((instance, selected_backend)) = activate_warm_instance(state, &pool_key).await? - { - state.metrics.inc(&state.metrics.pool_hits); - return json_created(&CreateInstanceResp { - start_path: instance.start_path, - instance, - decision, - selected_backend, - }); - } - state.metrics.inc(&state.metrics.pool_misses); - } - - let start_path = StartPath::Cold; - let mut instance = SandboxInstance::new( - runtime_backend, - decision.workload_class, - req.image_digest.clone(), - start_path, - decision.policy_name.clone(), - ); - instance.transition(SandboxState::Creating)?; - let operation_lock = state.operation_lock(instance.id); - let _operation = operation_lock.lock().await; - - let (binary_path, rootfs_size, mem_size) = { - let cfg = state - .config - .lock() - .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))?; - ( - cfg.backends - .get(state.active_backend.as_str()) - .cloned() - .unwrap_or_default(), - cfg.storage.rootfs_size, - cfg.storage.mem_size, - ) - }; - // Publish ownership before allocation. A restart can now discover this - // stable ID and release either an absent slot or a completed allocation. - instance.persist(&state.state_dir)?; - if let Some(error) = retain_instance_state(state, instance.clone()) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "create {}: {error}", - instance.id - ))); - } - let storage = match state - .storage - .acquire(&AcquireOpts { - instance_id: instance.id.to_string(), - rootfs_size, - mem_size, + let binary_path = state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(state.active_backend.as_str()) + .cloned() + .unwrap_or_default(); + + let created = state + .manager + .create(CreateSandbox { + decision: decision.clone(), + image_digest: req.image_digest, + runtime_backend, + binary_path, }) - .await - { - Ok(storage) => storage, - Err(error) => { - let (source, residual) = error.into_parts(); - return Err(retain_failed_acquire( - state, - &mut instance, - residual, - source.into(), - )); - } - }; - crate::failpoint::pause("create-after-storage-acquire").await; - - let work_dir = state.state_dir.join(instance.id.to_string()); - let spawner = match state.spawner_for(state.active_backend) { - Some(spawner) => spawner, - None => { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(format!( - "active backend {} has no registered spawner", - state.active_backend - )), - ) - .await); - } - }; - if let Err(error) = spawner.prepare_spawn(&work_dir).await { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - error.into(), - ) - .await); - } - - instance.backend_ownership = BackendOwnership::Starting; - if let Err(error) = instance.persist(&state.state_dir) { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - error.into(), - ) - .await); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - instance.backend_ownership = BackendOwnership::NotStarted; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - false, - BlazeDaemonError::Internal(error), - ) - .await); - } - - let spawn = match crate::failpoint::backend("create-spawn") { - Ok(()) => { - spawner - .spawn(SpawnRequest { - instance_id: instance.id, - run_dir: work_dir, - binary_path, - storage: storage.clone(), - backend: decision.backend.clone(), - vm: decision.vm.clone(), - }) - .await - } - Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), - }; - let actual_backend = match spawn { - Ok(backend_instance) => { - instance.backend_ownership = BackendOwnership::Running; - let real_backend = backend_instance.backend(); - let mut backend_instance = Some(backend_instance); - let registered = match state.backend_instances.lock() { - Ok(mut instances) => { - instances.insert( - instance.id, - backend_instance - .take() - .expect("backend instance is present"), - ); - true - } - Err(_) => false, - }; - if !registered { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - backend_instance, - false, - BlazeDaemonError::Internal("backend_instances lock poisoned".to_string()), - ) - .await); - } - real_backend - } - Err(error) => { - let (source, backend) = error.into_parts(); - instance.backend_ownership = if backend.is_some() { - BackendOwnership::Running - } else { - BackendOwnership::Stopped - }; - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - backend, - false, - source.into(), - ) - .await); - } - }; - if let Err(error) = instance.transition(SandboxState::Running) { - return Err( - cleanup_failed_create(state, &mut instance, storage, None, true, error.into()).await, - ); - } - if let Err(error) = crate::failpoint::state("create-state-commit") - .and_then(|_| instance.persist(&state.state_dir).map_err(Into::into)) - { - return Err(cleanup_failed_create(state, &mut instance, storage, None, true, error).await); - } - - let inserted = match state.instances.lock() { - Ok(mut instances) => { - instances.insert(instance.id, instance.clone()); - true - } - Err(_) => false, - }; - if !inserted { - return Err(cleanup_failed_create( - state, - &mut instance, - storage, - None, - true, - BlazeDaemonError::Internal("instances lock poisoned".to_string()), - ) - .await); - } - state.metrics.inc(&state.metrics.instances_created); - + .await?; json_created(&CreateInstanceResp { - instance, + start_path: created.instance.start_path, + instance: created.instance, decision, - start_path, - selected_backend: actual_backend, + selected_backend: created.selected_backend, }) } -async fn activate_warm_instance( - state: &Arc, - key: &PoolKey, -) -> Result> { - let candidate = state - .pool - .lock() - .map_err(|_| BlazeDaemonError::Internal("pool lock poisoned".into()))? - .lookup(key); - let Some(id) = candidate else { - return Ok(None); - }; - let operation_lock = state.operation_lock(id); - let _operation = operation_lock.lock().await; - - let instance = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .get(&id) - .cloned(); - let backend = state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .get(&id) - .cloned(); - - let invalid_reason = match (&instance, &backend) { - (None, _) => Some("lifecycle metadata is missing".to_string()), - (_, None) => Some("backend owner is missing".to_string()), - (Some(instance), Some(_)) if instance.state != SandboxState::Warm => { - Some(format!("lifecycle state is {}", instance.state)) - } - (Some(instance), Some(backend)) if backend.backend() != instance.backend => Some(format!( - "backend owner is {}, metadata is {}", - backend.backend(), - instance.backend - )), - (_, Some(backend)) => match backend.try_wait().await { - Ok(None) => None, - Ok(Some(status)) => Some(format!("backend exited: {status:?}")), - Err(error) => Some(format!("backend liveness check failed: {error}")), - }, - }; - - if let Some(reason) = invalid_reason { - quarantine_warm_instance(state, key, id, instance, backend, &reason).await; - return Ok(None); - } - - let original = instance.expect("validated warm metadata"); - match state.storage.reconstruct(&id.to_string()).await { - Ok(_) => {} - Err(error @ BlazeError::StorageIncomplete { .. }) => { - quarantine_warm_instance( - state, - key, - id, - Some(original), - backend, - &format!("storage validation failed: {error}"), - ) - .await; - return Ok(None); - } - Err(error) => { - return Err(restore_warm_claim(state, key, original, error.into())); - } - } - - crate::failpoint::pause("warm-before-state-commit").await; - let mut instance = original.clone(); - let selected_backend = backend.expect("validated warm backend").backend(); - if let Err(error) = instance - .transition(SandboxState::Creating) - .and_then(|_| instance.transition(SandboxState::Running)) - .and_then(|_| instance.persist(&state.state_dir)) - { - return Err(restore_warm_claim(state, key, original, error.into())); - } - state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .insert(id, instance.clone()); - Ok(Some((instance, selected_backend))) -} - -fn restore_warm_claim( - state: &Arc, - key: &PoolKey, - instance: SandboxInstance, - cause: BlazeDaemonError, -) -> BlazeDaemonError { - let id = instance.id; - let mut errors = Vec::new(); - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("restore warm state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance) { - errors.push(error); - } - match state.pool.lock() { - Ok(mut pool) => pool.restore_lookup(key.clone(), id), - Err(poisoned) => { - poisoned.into_inner().restore_lookup(key.clone(), id); - errors.push("pool lock poisoned while restoring warm claim".to_string()); - } - } - let details = if errors.is_empty() { - "warm claim restored for retry".to_string() - } else { - format!("warm claim restored with errors: {}", errors.join("; ")) - }; - BlazeDaemonError::RecoveryRequired(format!("{cause}; instance {id}: {details}")) -} - -async fn quarantine_warm_instance( - state: &Arc, - key: &PoolKey, - id: Uuid, - mut instance: Option, - backend: Option, - reason: &str, -) { - match state.pool.lock() { - Ok(mut pool) => pool.quarantine(key, id), - Err(poisoned) => poisoned.into_inner().quarantine(key, id), - } - tracing::warn!(instance = %id, reason, "warm instance validation failed"); - - let backend_stopped = match backend.as_ref() { - Some(backend) => match backend.kill().await { - Ok(()) => true, - Err(error) => { - tracing::error!(instance = %id, %error, "quarantined backend cleanup failed"); - false - } - }, - None => match instance.as_ref() { - Some(instance) - if matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) => - { - true - } - Some(instance) => match state.spawner_for(instance.backend) { - Some(spawner) => match spawner - .cleanup_orphan(id, &state.state_dir.join(id.to_string())) - .await - { - Ok(()) => true, - Err(error) => { - tracing::error!( - instance = %id, - %error, - "quarantined orphan cleanup failed" - ); - false - } - }, - None => { - tracing::error!( - instance = %id, - backend = %instance.backend, - "quarantined backend has no recovery spawner" - ); - false - } - }, - None => false, - }, - }; - if !backend_stopped { - return; - } - let Some(instance) = instance.as_mut() else { - tracing::error!( - instance = %id, - "quarantined lifecycle metadata missing; retaining storage" - ); - return; - }; - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.persist(&state.state_dir) { - tracing::error!(instance = %id, %error, "quarantined stop state commit failed"); - return; - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - tracing::error!(instance = %id, %error, "quarantined stop state retention failed"); - return; - } - if let Err(error) = state.storage.release_by_id(&id.to_string()).await { - tracing::error!(instance = %id, %error, "quarantined storage cleanup failed"); - return; - } - - if instance.state != SandboxState::Destroyed - && let Err(error) = instance.transition(SandboxState::Destroyed) - { - tracing::error!(instance = %id, %error, "quarantined lifecycle cleanup failed"); - return; - } - if let Err(error) = instance.persist(&state.state_dir) { - tracing::error!(instance = %id, %error, "quarantined state commit failed"); - return; - } - match state.instances.lock() { - Ok(mut instances) => { - instances.insert(id, instance.clone()); - } - Err(poisoned) => { - poisoned.into_inner().insert(id, instance.clone()); - } - } - match state.backend_instances.lock() { - Ok(mut instances) => { - instances.remove(&id); - } - Err(poisoned) => { - poisoned.into_inner().remove(&id); - } - } -} - -async fn cleanup_failed_create( - state: &Arc, - instance: &mut SandboxInstance, - storage: blaze_core::storage::StorageSlot, - backend: Option, - registered: bool, - original: BlazeDaemonError, -) -> BlazeDaemonError { - let mut cleanup_errors = Vec::new(); - let backend = if registered { - match state.backend_instances.lock() { - Ok(mut instances) => instances.remove(&instance.id), - Err(poisoned) => poisoned.into_inner().remove(&instance.id), - } - } else { - backend - }; - let mut backend_stopped = matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ); - if registered && backend.is_none() { - backend_stopped = false; - cleanup_errors.push("registered backend owner is missing".to_string()); - } - if let Some(backend) = backend.as_ref() { - match backend.kill().await { - Ok(()) => { - backend_stopped = true; - instance.backend_ownership = BackendOwnership::Stopped; - } - Err(error) => { - backend_stopped = false; - cleanup_errors.push(format!("backend termination failed: {error}")); - } - } - } - - let mut storage_released = false; - if backend_stopped { - match state.storage.release(storage).await { - Ok(()) => storage_released = true, - Err(error) => cleanup_errors.push(format!("storage release failed: {error}")), - } - } else { - cleanup_errors.push("storage retained until backend termination succeeds".to_string()); - } - - if backend_stopped && storage_released { - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.transition(SandboxState::Destroyed) { - let mut recovery_errors = vec![format!("lifecycle update failed: {error}")]; - if let Err(persist_error) = instance.persist(&state.state_dir) { - recovery_errors.push(format!("state persistence failed: {persist_error}")); - } - if let Some(retain_error) = retain_instance_state(state, instance.clone()) { - recovery_errors.push(retain_error); - } - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {}", - recovery_errors.join("; ") - )); - } - if let Err(error) = instance.persist(&state.state_dir) { - let mut recovery_errors = vec![format!("state persistence failed: {error}")]; - if let Some(retain_error) = retain_instance_state(state, instance.clone()) { - recovery_errors.push(retain_error); - } - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {}", - recovery_errors.join("; ") - )); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup completed but {error}" - )); - } - state.metrics.inc(&state.metrics.instances_destroyed); - return original; - } - - if let Some(backend) = backend - && let Some(error) = retain_backend_owner(state, instance.id, backend) - { - cleanup_errors.push(error); - } - if let Err(error) = instance.persist(&state.state_dir) { - cleanup_errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - cleanup_errors.push(error); - } - BlazeDaemonError::RecoveryRequired(format!( - "{original}; cleanup incomplete: {}", - cleanup_errors.join("; ") - )) -} - -fn retain_failed_acquire( - state: &Arc, - instance: &mut SandboxInstance, - residual: Option, - original: BlazeDaemonError, -) -> BlazeDaemonError { - if residual.is_some() { - let mut errors = Vec::new(); - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - errors.push(error); - } - let suffix = if errors.is_empty() { - "residual storage retained for destroy retry".to_string() - } else { - format!( - "residual storage retained with recovery errors: {}", - errors.join("; ") - ) - }; - return BlazeDaemonError::RecoveryRequired(format!( - "{original}; instance {}: {suffix}", - instance.id - )); - } - - let mut errors = Vec::new(); - instance.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = instance.transition(SandboxState::Destroyed) { - errors.push(format!("lifecycle update failed: {error}")); - } - if let Err(error) = instance.persist(&state.state_dir) { - errors.push(format!("state persistence failed: {error}")); - } - if let Some(error) = retain_instance_state(state, instance.clone()) { - errors.push(error); - } - if errors.is_empty() { - original - } else { - BlazeDaemonError::RecoveryRequired(format!( - "{original}; acquire rollback completed but {}", - errors.join("; ") - )) - } -} - -fn retain_backend_owner( - state: &Arc, - id: Uuid, - backend: crate::spawner::DynBackendInstance, -) -> Option { - match state.backend_instances.lock() { - Ok(mut instances) => { - instances.insert(id, backend); - None - } - Err(poisoned) => { - poisoned.into_inner().insert(id, backend); - Some("backend owner retained in poisoned runtime map".to_string()) - } - } -} - -fn retain_instance_state(state: &Arc, instance: SandboxInstance) -> Option { - match state.instances.lock() { - Ok(mut instances) => { - instances.insert(instance.id, instance); - None - } - Err(poisoned) => { - poisoned.into_inner().insert(instance.id, instance); - Some("instance state retained in poisoned lifecycle map".to_string()) - } - } -} - async fn checkpoint(state: &Arc, id: &str) -> Result>> { let uuid = parse_uuid(id)?; - let operation_lock = state.operation_lock(uuid); - let _operation = operation_lock.lock().await; - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - map.get(&uuid) - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; + let _operation = state + .manager + .lock_quiescent_state(uuid, SandboxState::Running) + .await?; Err(BlazeDaemonError::UnsupportedOperation(format!( "instance {uuid} cannot be checkpointed until its backend and storage state can be captured" @@ -922,14 +276,10 @@ async fn checkpoint(state: &Arc, id: &str) -> Result, id: &str) -> Result>> { let uuid = parse_uuid(id)?; - let operation_lock = state.operation_lock(uuid); - let _operation = operation_lock.lock().await; - let map = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))?; - map.get(&uuid) - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; + let _operation = state + .manager + .lock_quiescent_state(uuid, SandboxState::Running) + .await?; Err(BlazeDaemonError::UnsupportedOperation(format!( "instance {uuid} cannot be reset until its backend can reset runtime and storage state" @@ -968,93 +318,10 @@ async fn return_to_pool_for_test(state: &Arc, id: &str) -> Result<( async fn destroy_instance(state: &Arc, id: &str) -> Result>> { let uuid = parse_uuid(id)?; - let operation_lock = state.operation_lock(uuid); - let _operation = operation_lock.lock().await; - let mut original = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .get(&uuid) - .cloned() - .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {uuid}")))?; - let backend = state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .get(&uuid) - .cloned(); - - let stop_result = match crate::failpoint::backend("destroy-kill") { - Ok(()) => { - if let Some(backend) = backend.as_ref() { - backend.kill().await - } else if matches!( - original.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) { - Ok(()) - } else { - match state.spawner_for(original.backend) { - Some(spawner) => { - spawner - .cleanup_orphan(uuid, &state.state_dir.join(uuid.to_string())) - .await - } - None => Err(BlazeError::BackendError { - msg: format!( - "no recovery spawner registered for persisted backend {}", - original.backend - ), - }), - } - } - } - Err(error) => Err(error), - }; - if let Err(error) = stop_result { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend termination failed: {error}; owner and storage retained" - ))); - } - - original.backend_ownership = BackendOwnership::Stopped; - if let Err(error) = original.persist(&state.state_dir) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but stop state persistence failed: {error}; storage retained" - ))); - } - if let Some(error) = retain_instance_state(state, original.clone()) { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but lifecycle retention failed: {error}; storage retained" - ))); - } - - if let Err(error) = state.storage.release_by_id(&uuid.to_string()).await { - return Err(BlazeDaemonError::RecoveryRequired(format!( - "destroy {uuid}: backend stopped but storage release failed: {error}; lifecycle retained for retry" - ))); - } - - let mut destroyed = original; - if destroyed.state != SandboxState::Destroyed { - destroyed.transition(SandboxState::Destroyed)?; - } - destroyed.persist(&state.state_dir)?; - state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .insert(uuid, destroyed.clone()); - state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .remove(&uuid); - - state.metrics.inc(&state.metrics.instances_destroyed); + state.manager.destroy(uuid).await?; json_ok(&json!({ "destroyed": true, - "instance_id": destroyed.id, + "instance_id": uuid, })) } @@ -1065,36 +332,14 @@ async fn destroy_instance(state: &Arc, id: &str) -> Result, budget: Duration) -> Result<()> { - let mut ids = state - .instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("instances lock poisoned".into()))? - .iter() - .filter_map(|(id, instance)| { - let clean_terminal = instance.state == SandboxState::Destroyed - && matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ); - (!clean_terminal).then_some(*id) - }) - .collect::>(); - ids.extend( - state - .backend_instances - .lock() - .map_err(|_| BlazeDaemonError::Internal("backend_instances lock poisoned".into()))? - .keys() - .copied(), - ); + let ids = state.manager.owned_instance_ids()?; let deadline = Instant::now() + budget; let mut tasks = JoinSet::new(); let mut task_owners = HashMap::new(); for id in ids { let state = state.clone(); - let task = - tasks.spawn(async move { destroy_instance(&state, &id.to_string()).await.map(|_| ()) }); + let task = tasks.spawn(async move { state.manager.destroy(id).await.map(|_| ()) }); task_owners.insert(task.id(), id); } @@ -1448,11 +693,14 @@ mod tests { use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::time::Duration; use async_trait::async_trait; - use blaze_core::backend::BackendKind; + use blaze_core::BlazeError; + use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; + use blaze_core::lifecycle::{BackendOwnership, OperationKind}; use blaze_core::policy::{ BackendConfigs, FallbackOnMissingHook, PolicyEngine, PolicyFile, PolicyHooks, PolicyMatch, PolicyPool, PolicySelect, ResetMode, WorkloadClass, @@ -1466,8 +714,8 @@ mod tests { use crate::file_provider::FileStorageProvider; use crate::spawner::{ - BackendInstance, BackendSpawner, DynBackendInstance, DynSpawner, MockSpawner, SpawnFailure, - SpawnResult, SpawnerRegistry, + BackendInstance, BackendSpawner, BubblewrapSpawner, DynBackendInstance, DynSpawner, + MockSpawner, SpawnFailure, SpawnResult, SpawnerRegistry, }; use crate::state::ServerState; @@ -1657,6 +905,10 @@ mod tests { let instance = SandboxInstance::load(&self.state_dir, id).expect("ownership published"); assert_eq!(instance.state, SandboxState::Creating); assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); self.observed.store(true, Ordering::Release); self.inner.acquire(opts).await } @@ -1798,15 +1050,18 @@ mod tests { .lock() .expect("instances") .insert(id, instance); - state.backend_instances.lock().expect("owners").insert( - id, - Arc::new(ShutdownOwner { - instance_id: id, - behavior, - attempts, - active, - }), - ); + state + .manager + .insert_backend_owner( + id, + Arc::new(ShutdownOwner { + instance_id: id, + behavior, + attempts, + active, + }), + ) + .expect("retain backend owner"); id } @@ -1874,14 +1129,234 @@ mod tests { } } - /// When multiple backend binaries exist on disk but the daemon probed - /// Firecracker at boot, only Firecracker should be reported available - /// and selected — even if policy prioritizes bubblewrap higher. - #[tokio::test] - async fn availability_constrained_to_active_backend() { - // Create temp files to simulate both binaries existing. - let tmp = std::env::temp_dir().join("blaze-test-active-backend"); - let _ = std::fs::create_dir_all(&tmp); + struct SelectiveCleanupSpawner { + failed_id: Uuid, + cleanup_count: Arc, + } + + #[async_trait] + impl BackendSpawner for SelectiveCleanupSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + MockSpawner.spawn(request).await + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + self.cleanup_count.fetch_add(1, Ordering::AcqRel); + if instance_id == self.failed_id { + return Err(BlazeError::BackendError { + msg: "cleanup deferred".into(), + }); + } + Ok(()) + } + } + + struct CountingOwner { + instance_id: Uuid, + kill_count: Arc, + killed: AtomicBool, + } + + #[async_trait] + impl BackendInstance for CountingOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(self.killed.load(Ordering::Acquire).then_some(SpawnResult { + instance_id: self.instance_id, + exit_code: Some(0), + signal: None, + })) + } + + async fn kill(&self) -> blaze_core::Result<()> { + if !self.killed.swap(true, Ordering::AcqRel) { + self.kill_count.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } + } + + struct CountingSpawner { + kill_count: Arc, + orphan_cleanup_count: Arc, + } + + #[async_trait] + impl BackendSpawner for CountingSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + Ok(Arc::new(CountingOwner { + instance_id: request.instance_id, + kill_count: self.kill_count.clone(), + killed: AtomicBool::new(false), + })) + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + _instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + self.orphan_cleanup_count.fetch_add(1, Ordering::AcqRel); + Ok(()) + } + } + + struct CountingStorage { + inner: FileStorageProvider, + release_count: Arc, + } + + #[async_trait] + impl StorageProvider for CountingStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.release_count.fetch_add(1, Ordering::AcqRel); + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + self.release_count.fetch_add(1, Ordering::AcqRel); + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + struct SelectiveHangingStorage { + inner: FileStorageProvider, + stalled_id: Uuid, + } + + #[async_trait] + impl StorageProvider for SelectiveHangingStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + if instance_id == self.stalled_id.to_string() { + return std::future::pending().await; + } + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + + #[cfg(feature = "test-failpoints")] + fn counting_state( + temp: &tempfile::TempDir, + ) -> ( + Arc, + Arc, + Arc, + Arc, + ) { + let config = test_config(temp); + let kill_count = Arc::new(AtomicUsize::new(0)); + let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage: Arc = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: orphan_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + (state, kill_count, orphan_cleanup_count, release_count) + } + + /// When multiple backend binaries exist on disk but the daemon probed + /// Firecracker at boot, only Firecracker should be reported available + /// and selected — even if policy prioritizes bubblewrap higher. + #[tokio::test] + async fn availability_constrained_to_active_backend() { + // Create temp files to simulate both binaries existing. + let tmp = std::env::temp_dir().join("blaze-test-active-backend"); + let _ = std::fs::create_dir_all(&tmp); let fc_bin = tmp.join("firecracker"); let bwrap_bin = tmp.join("bwrap"); std::fs::write(&fc_bin, b"fake-fc").unwrap(); @@ -2002,13 +1477,7 @@ mod tests { std::fs::read(state_path).expect("persisted state"), persisted_before ); - assert!( - state - .backend_instances - .lock() - .expect("owners") - .contains_key(&uuid) - ); + assert!(state.manager.backend_owner(uuid).is_some()); } #[tokio::test] @@ -2047,15 +1516,166 @@ mod tests { "sha256:ownership-test".into(), ); assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_rejects_an_unfinished_lifecycle_journal() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let journal = { + let mut instances = state.instances.lock().expect("instances"); + let instance = instances.get_mut(&uuid).expect("instance"); + instance + .begin_operation(OperationKind::Create) + .expect("begin unfinished operation"); + instance.persist(&state.state_dir).expect("persist journal"); + instance.operation.clone().expect("journal") + }; + + let error = checkpoint(&state, id) + .await + .expect_err("unfinished lifecycle work must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].operation, + Some(journal) + ); + assert_eq!( + SandboxInstance::load(&state.state_dir, uuid) + .expect("persisted instance") + .operation, + state.instances.lock().expect("instances")[&uuid].operation + ); + } + + #[tokio::test] + async fn checkpoint_rejects_a_non_running_lifecycle_state() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state.manager.destroy(uuid).await.expect("destroy"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint must require a running instance"); + + assert!(matches!(error, BlazeDaemonError::Conflict(_))); + assert_eq!(error.status_code(), 409); + } + + #[tokio::test] + async fn quiescent_state_guard_serializes_later_destroy() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("id")).expect("uuid"); + let guard = state + .manager + .lock_quiescent_state(id, SandboxState::Running) + .await + .expect("quiescent running state"); + let destroy_state = state.clone(); + let mut destroy = tokio::spawn(async move { destroy_state.manager.destroy(id).await }); + assert!( - state - .backend_instances - .lock() - .expect("owners") - .contains_key(&uuid) + tokio::time::timeout(Duration::from_millis(20), &mut destroy) + .await + .is_err(), + "destroy must wait while a lifecycle operation holds the guard" + ); + drop(guard); + + assert!(destroy.await.expect("destroy task").expect("destroy")); + assert_eq!( + state.manager.get(id).expect("instance").state, + SandboxState::Destroyed ); } + #[tokio::test] + async fn manager_cleanup_releases_tracked_runtime_resources() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"); + { + let mut instances = state.instances.lock().expect("instances"); + let instance = instances.get_mut(&id).expect("instance"); + instance + .transition(SandboxState::Destroyed) + .expect("persisted terminal state"); + instance.backend_ownership = BackendOwnership::Stopped; + instance.finish_operation(); + instance.persist(&state.state_dir).expect("persist"); + } + + let report = state + .manager + .cleanup_owned_instances_with_timeout(Duration::from_secs(1)) + .await; + + assert!(report.failures.is_empty()); + assert_eq!( + state.manager.get(id).expect("instance").state, + SandboxState::Destroyed + ); + assert!(state.manager.backend_owner(id).is_none()); + assert!(!instances_dir.join(id.to_string()).exists()); + } + #[tokio::test] async fn warm_claim_validates_runtime_and_quarantines_dead_owner() { let temp = tempfile::tempdir().expect("temp"); @@ -2143,11 +1763,8 @@ mod tests { .await .expect("return live owner"); let owner = state - .backend_instances - .lock() - .expect("owners") - .get(&Uuid::parse_str(&id).expect("uuid")) - .cloned() + .manager + .backend_owner(Uuid::parse_str(&id).expect("uuid")) .expect("owner"); owner.kill().await.expect("simulate backend exit"); @@ -2231,26 +1848,14 @@ mod tests { .expect("shutdown cleanup"); assert_eq!( - state - .instances - .lock() - .expect("instances") - .get(&id) - .expect("instance") - .state, + state.manager.get(id).expect("instance").state, SandboxState::Destroyed ); - assert!( - !state - .backend_instances - .lock() - .expect("backend owners") - .contains_key(&id) - ); + assert!(state.manager.backend_owner(id).is_none()); assert!(!instances_dir.join(id.to_string()).exists()); } - #[tokio::test] + #[tokio::test(start_paused = true)] async fn daemon_shutdown_joins_all_owners_within_one_budget() { let temp = tempfile::tempdir().expect("temp"); let config = test_config(&temp); @@ -2319,13 +1924,7 @@ mod tests { let message = error.to_string(); for id in [failed, stalled_a, stalled_b] { assert!(message.contains(&id.to_string())); - assert!( - state - .backend_instances - .lock() - .expect("owners") - .contains_key(&id) - ); + assert!(state.manager.backend_owner(id).is_some()); assert!(instances_dir.join(id.to_string()).is_dir()); } assert_eq!( @@ -2338,13 +1937,7 @@ mod tests { .state, SandboxState::Destroyed ); - assert!( - !state - .backend_instances - .lock() - .expect("owners") - .contains_key(&complete) - ); + assert!(state.manager.backend_owner(complete).is_none()); assert!(!instances_dir.join(complete.to_string()).exists()); } @@ -2369,6 +1962,7 @@ mod tests { let id = cold["instance"]["id"].as_str().expect("id").to_string(); assert_eq!(cold["instance"]["backend"], "mock"); assert_eq!(cold["selected_backend"], "mock"); + assert!(cold["instance"]["operation"].is_null()); return_to_pool_for_test(&state, &id) .await @@ -2378,6 +1972,7 @@ mod tests { assert_eq!(warm["instance"]["backend"], "mock"); assert_eq!(warm["selected_backend"], "mock"); assert_eq!(warm["start_path"], "warm"); + assert!(warm["instance"]["operation"].is_null()); } #[tokio::test] @@ -2409,15 +2004,14 @@ mod tests { .next() .cloned() .expect("retained lifecycle"); + assert_eq!(instance.state, SandboxState::RecoveryRequired); assert_eq!(instance.backend_ownership, BackendOwnership::Running); - assert!(instances_dir.join(instance.id.to_string()).is_dir()); - assert!( - state - .backend_instances - .lock() - .expect("owners") - .contains_key(&instance.id) + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) ); + assert!(instances_dir.join(instance.id.to_string()).is_dir()); + assert!(state.manager.backend_owner(instance.id).is_some()); destroy_instance(&state, &instance.id.to_string()) .await @@ -2671,6 +2265,64 @@ mod tests { ); } + #[cfg(feature = "test-failpoints")] + async fn assert_warm_state_commit_failure_restores_claim(failpoint: &'static str) { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, true); + let request = test_request(); + let cold = created_json(&state, &request).await; + let id = cold["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + return_to_pool_for_test(&state, &id).await.expect("warm"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let key = PoolKey::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + ); + + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let error = hook + .run(create_instance(&state, &request)) + .await + .expect_err("state commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let restored = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(restored.state, SandboxState::Warm); + assert!(restored.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted warm state"); + assert_eq!(persisted.state, SandboxState::Warm); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + let retained_owner = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained_owner)); + assert_eq!(state.pool.lock().expect("pool").stats(&key).warm_count, 1); + assert!(retained_owner.try_wait().await.expect("liveness").is_none()); + + let retried = created_json(&state, &request).await; + assert_eq!(retried["instance"]["id"], id); + assert_eq!(retried["start_path"], "warm"); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted running state"); + assert_eq!(persisted.state, SandboxState::Running); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn warm_intent_commit_failure_restores_the_claim() { + assert_warm_state_commit_failure_restores_claim("warm-intent-state-commit").await; + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn warm_final_commit_failure_restores_the_claim() { + assert_warm_state_commit_failure_restores_claim("warm-final-state-commit").await; + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn failure_hooks_drive_create_and_destroy_compensation() { @@ -2711,10 +2363,9 @@ mod tests { assert_eq!(commit_instance.state, SandboxState::Destroyed); assert!( commit_state - .backend_instances - .lock() - .expect("owners") - .is_empty() + .manager + .backend_owner(commit_instance.id) + .is_none() ); let destroy_temp = tempfile::tempdir().expect("temp"); @@ -2727,13 +2378,16 @@ mod tests { .await .expect_err("kill boundary"); let uuid = Uuid::parse_str(&id).expect("uuid"); - assert!( - destroy_state - .backend_instances - .lock() - .expect("owners") - .contains_key(&uuid) + let failed_destroy = destroy_state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(failed_destroy.state, SandboxState::RecoveryRequired); + assert_eq!( + failed_destroy + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Destroy) ); + assert!(destroy_state.manager.backend_owner(uuid).is_some()); destroy_instance(&destroy_state, &id) .await .expect("destroy retry"); @@ -2742,19 +2396,161 @@ mod tests { let release_state = mock_state(&release_temp, false); let created = created_json(&release_state, &request).await; let id = created["instance"]["id"].as_str().expect("id").to_string(); - let release_hook = crate::failpoint::TestFailpoint::new(&["storage-release"]); - release_hook - .run(destroy_instance(&release_state, &id)) - .await - .expect_err("release boundary"); + let release_hook = crate::failpoint::TestFailpoint::new(&["storage-release"]); + release_hook + .run(destroy_instance(&release_state, &id)) + .await + .expect_err("release boundary"); + let uuid = Uuid::parse_str(&id).expect("uuid"); + assert_eq!( + release_state.instances.lock().expect("instances")[&uuid].backend_ownership, + BackendOwnership::Stopped + ); + assert_eq!( + release_state.instances.lock().expect("instances")[&uuid] + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + destroy_instance(&release_state, &id) + .await + .expect("release retry"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_intent_failure_does_not_touch_owned_resources() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-intent-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("intent failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 0); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert!(retained.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(persisted.operation.is_none()); + assert!(temp.path().join("instances").join(&id).is_dir()); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 1); + assert_eq!( + state.instances.lock().expect("instances")[&uuid].state, + SandboxState::Destroyed + ); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_stop_commit_failure_retains_storage_for_retry() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-stop-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("stop commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 0); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!(retained.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + persisted.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!(temp.path().join("instances").join(&id).is_dir()); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 1); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn destroy_final_commit_failure_retains_retryable_metadata() { + let temp = tempfile::tempdir().expect("temp"); + let (state, kill_count, orphan_cleanup_count, release_count) = counting_state(&temp); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); let uuid = Uuid::parse_str(&id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["destroy-final-state-commit"]); + + let error = hook + .run(destroy_instance(&state, &id)) + .await + .expect_err("final commit failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 1); + assert!(!temp.path().join("instances").join(&id).exists()); + let retained = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!(retained.backend_ownership, BackendOwnership::Stopped); assert_eq!( - release_state.instances.lock().expect("instances")[&uuid].backend_ownership, - BackendOwnership::Stopped + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) ); - destroy_instance(&release_state, &id) - .await - .expect("release retry"); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted recovery state"); + assert_eq!(persisted.state, SandboxState::RecoveryRequired); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + persisted.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + + destroy_instance(&state, &id).await.expect("destroy retry"); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 2); + let destroyed = state.instances.lock().expect("instances")[&uuid].clone(); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted destroyed state"); + assert_eq!(persisted.state, SandboxState::Destroyed); + assert!(persisted.operation.is_none()); } #[cfg(feature = "test-failpoints")] @@ -2780,8 +2576,12 @@ mod tests { .next() .cloned() .expect("recovery record"); - assert_eq!(instance.state, SandboxState::Creating); + assert_eq!(instance.state, SandboxState::RecoveryRequired); assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); assert!( temp.path() .join("instances") @@ -2831,6 +2631,10 @@ mod tests { let id = instance.id; assert_eq!(instance.state, SandboxState::Creating); assert_eq!(instance.backend_ownership, BackendOwnership::NotStarted); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); assert!( config .daemon @@ -2907,6 +2711,14 @@ mod tests { .await }); pause_hook.wait_until_paused().await; + let uuid = Uuid::parse_str(&id).expect("uuid"); + assert_eq!( + state.instances.lock().expect("instances")[&uuid] + .operation + .as_ref() + .map(|operation| operation.kind), + Some(OperationKind::Create) + ); let destroy_state = state.clone(); let destroy_id = id.clone(); @@ -2921,10 +2733,494 @@ mod tests { .expect("activation task") .expect("activation"); destroy.await.expect("destroy task").expect("destroy"); - let uuid = Uuid::parse_str(&id).expect("uuid"); assert_eq!( state.instances.lock().expect("instances")[&uuid].state, SandboxState::Destroyed ); } + + #[tokio::test] + async fn startup_reconciliation_repairs_anomalous_destroyed_records() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + + let clean_stopped_id = Uuid::new_v4(); + let clean_not_started_id = Uuid::new_v4(); + let legacy_id = Uuid::new_v4(); + let active_id = Uuid::new_v4(); + for (id, ownership, active_operation) in [ + (clean_stopped_id, BackendOwnership::Stopped, false), + (clean_not_started_id, BackendOwnership::NotStarted, false), + (legacy_id, BackendOwnership::Unknown, false), + (active_id, BackendOwnership::Running, true), + ] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:destroyed-reconcile".into(), + StartPath::Cold, + "destroyed-reconcile-test".into(), + ); + instance.id = id; + if active_operation { + instance + .begin_operation(OperationKind::Create) + .expect("begin interrupted create"); + } + instance + .transition(SandboxState::Destroyed) + .expect("destroyed"); + instance.backend_ownership = ownership; + instance.persist(&config.daemon.state_dir).expect("persist"); + + if id == legacy_id || id == active_id { + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + } + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(SelectiveCleanupSpawner { + failed_id: legacy_id, + cleanup_count: cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage.clone(), + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, legacy_id); + assert_eq!(cleanup_count.load(Ordering::Acquire), 2); + assert_eq!(release_count.load(Ordering::Acquire), 1); + let retryable = state.manager.get(legacy_id).expect("retryable instance"); + assert_eq!(retryable.state, SandboxState::Destroyed); + assert_eq!(retryable.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + retryable.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!( + config + .storage + .instances_dir + .join(legacy_id.to_string()) + .is_dir() + ); + + drop(state); + let retry_cleanup_count = Arc::new(AtomicUsize::new(0)); + let restarted = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(RecordingSpawner { + cleanup_count: retry_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + let retry_report = restarted.manager.reconcile_startup().await; + + assert_eq!(retry_report.attempted, 1); + assert_eq!(retry_report.completed, 1); + assert!(retry_report.failures.is_empty()); + assert_eq!(retry_cleanup_count.load(Ordering::Acquire), 1); + assert_eq!(release_count.load(Ordering::Acquire), 2); + for id in [clean_stopped_id, legacy_id, active_id] { + let instance = restarted.manager.get(id).expect("instance"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert_eq!(instance.backend_ownership, BackendOwnership::Stopped); + assert!(instance.operation.is_none()); + } + let clean_not_started = restarted + .manager + .get(clean_not_started_id) + .expect("clean not-started instance"); + assert_eq!(clean_not_started.state, SandboxState::Destroyed); + assert_eq!( + clean_not_started.backend_ownership, + BackendOwnership::NotStarted + ); + assert!(clean_not_started.operation.is_none()); + assert!( + !config + .storage + .instances_dir + .join(legacy_id.to_string()) + .exists() + ); + assert!( + !config + .storage + .instances_dir + .join(active_id.to_string()) + .exists() + ); + } + + #[tokio::test(start_paused = true)] + async fn startup_reconciliation_times_out_one_record_and_continues() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let stalled_id = Uuid::new_v4(); + let completed_id = Uuid::new_v4(); + let inner = FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ); + for id in [stalled_id, completed_id] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:bounded-reconcile".into(), + StartPath::Cold, + "bounded-reconcile-test".into(), + ); + instance.id = id; + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.persist(&config.daemon.state_dir).expect("persist"); + inner + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + let storage: Arc = + Arc::new(SelectiveHangingStorage { inner, stalled_id }); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let report = state + .manager + .cleanup_owned_instances_with_timeout(Duration::from_millis(20)) + .await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, stalled_id); + assert!(report.failures[0].error.contains("20 ms")); + let stalled = state.manager.get(stalled_id).expect("stalled record"); + assert_eq!(stalled.state, SandboxState::RecoveryRequired); + assert_eq!(stalled.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + stalled.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert_eq!( + state.manager.get(completed_id).expect("completed").state, + SandboxState::Destroyed + ); + assert!( + config + .storage + .instances_dir + .join(stalled_id.to_string()) + .is_dir() + ); + assert!( + !config + .storage + .instances_dir + .join(completed_id.to_string()) + .exists() + ); + + drop(state); + let retry_storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + retry_storage, + ); + let retry_report = restarted.manager.reconcile_startup().await; + + assert_eq!(retry_report.attempted, 1); + assert_eq!(retry_report.completed, 1); + assert!(retry_report.failures.is_empty()); + let recovered = restarted.manager.get(stalled_id).expect("recovered record"); + assert_eq!(recovered.state, SandboxState::Destroyed); + assert_eq!(recovered.backend_ownership, BackendOwnership::Stopped); + assert!(recovered.operation.is_none()); + } + + #[tokio::test] + async fn startup_reconciliation_continues_after_one_cleanup_failure() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let failed_id = Uuid::new_v4(); + let completed_id = Uuid::new_v4(); + for id in [failed_id, completed_id] { + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:reconcile".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + instance.id = id; + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance.persist(&config.daemon.state_dir).expect("persist"); + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(SelectiveCleanupSpawner { + failed_id, + cleanup_count: cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 1); + assert_eq!(report.failures.len(), 1); + assert_eq!(report.failures[0].instance_id, failed_id); + assert_eq!(cleanup_count.load(Ordering::Acquire), 2); + assert_eq!( + state.instances.lock().expect("instances")[&failed_id].state, + SandboxState::RecoveryRequired + ); + assert_eq!( + state.instances.lock().expect("instances")[&completed_id].state, + SandboxState::Destroyed + ); + assert!( + config + .storage + .instances_dir + .join(failed_id.to_string()) + .is_dir() + ); + assert!( + !config + .storage + .instances_dir + .join(completed_id.to_string()) + .exists() + ); + let created = created_json(&state, &test_request()).await; + assert_eq!(created["instance"]["state"], "running"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn startup_reconciliation_cleans_pre_spawn_backend_ownership() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let mut instance = SandboxInstance::new( + BackendKind::Bubblewrap, + WorkloadClass::AgentTool, + "sha256:pre-spawn".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + let id = instance.id; + let run_dir = config.daemon.state_dir.join(id.to_string()); + let spawner = Arc::new(BubblewrapSpawner); + spawner + .prepare_spawn(&run_dir) + .await + .expect("persist pre-spawn handoff"); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + instance.backend_ownership = BackendOwnership::Starting; + instance + .persist(&config.daemon.state_dir) + .expect("persist starting ownership"); + storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + assert!( + std::fs::read_to_string(run_dir.join("backend.pid")) + .expect("pre-spawn handoff") + .is_empty(), + "crash occurred after the empty handoff was persisted but before spawn" + ); + + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Bubblewrap, false), + spawners(BackendKind::Bubblewrap, spawner), + BackendKind::Bubblewrap, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); + assert_eq!( + state.instances.lock().expect("instances")[&id].state, + SandboxState::Destroyed + ); + assert!(!config.storage.instances_dir.join(id.to_string()).exists()); + assert!(run_dir.join("backend.stopped").is_file()); + } + + #[tokio::test] + async fn startup_reconciliation_skips_cleanup_for_known_stopped_states() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let release_count = Arc::new(AtomicUsize::new(0)); + let storage: Arc = Arc::new(CountingStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + release_count: release_count.clone(), + }); + let not_started_id = Uuid::new_v4(); + let stopped_id = Uuid::new_v4(); + + let mut not_started = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:not-started".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + not_started.id = not_started_id; + not_started + .transition(SandboxState::Creating) + .expect("creating"); + not_started + .persist(&config.daemon.state_dir) + .expect("persist"); + + let mut stopped = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:stopped".into(), + StartPath::Cold, + "reconcile-test".into(), + ); + stopped.id = stopped_id; + stopped + .transition(SandboxState::Creating) + .expect("creating"); + stopped.transition(SandboxState::Running).expect("running"); + stopped.backend_ownership = BackendOwnership::Stopped; + stopped.persist(&config.daemon.state_dir).expect("persist"); + + for id in [not_started_id, stopped_id] { + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage"); + } + let kill_count = Arc::new(AtomicUsize::new(0)); + let orphan_cleanup_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: orphan_cleanup_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + + let report = state.manager.reconcile_startup().await; + + assert_eq!(report.attempted, 2); + assert_eq!(report.completed, 2); + assert!(report.failures.is_empty()); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert_eq!(orphan_cleanup_count.load(Ordering::Acquire), 0); + assert_eq!(release_count.load(Ordering::Acquire), 2); + assert_eq!( + state.instances.lock().expect("instances")[¬_started_id].state, + SandboxState::Destroyed + ); + assert_eq!( + state.instances.lock().expect("instances")[&stopped_id].state, + SandboxState::Destroyed + ); + } } diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index c5534681d7..2a30ddf85f 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -191,6 +191,20 @@ pub async fn run(config_path: &Path) -> Result<()> { active_backend, storage, )?); + let reconciliation = state.manager.reconcile_startup().await; + tracing::info!( + attempted = reconciliation.attempted, + completed = reconciliation.completed, + failed = reconciliation.failures.len(), + "startup sandbox reconciliation completed" + ); + for failure in reconciliation.failures { + tracing::warn!( + instance = %failure.instance_id, + error = %failure.error, + "sandbox remains recovery-required after startup reconciliation" + ); + } let listener = DaemonSocket::bind(daemon_lock).await?; tracing::info!(socket = %socket_path.display(), "blaze UDS API listening"); diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index d3c9b60e20..560783115c 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -51,6 +51,9 @@ pub enum BlazeDaemonError { #[error("not found: {0}")] NotFound(String), + #[error("conflict: {0}")] + Conflict(String), + #[error("unsupported operation: {0}")] UnsupportedOperation(String), @@ -96,12 +99,14 @@ impl BlazeDaemonError { match self { BlazeDaemonError::BadRequest(_) | BlazeDaemonError::RequestBody(_) => 400, BlazeDaemonError::NotFound(_) => 404, + BlazeDaemonError::Conflict(_) => 409, BlazeDaemonError::PayloadTooLarge { .. } => 413, BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::RecoveryRequired(_) => 500, BlazeDaemonError::HttpStatus { status, .. } => *status, BlazeDaemonError::Core(blaze_core::BlazeError::PolicyEvalError { .. }) | BlazeDaemonError::Core(blaze_core::BlazeError::InvalidStateTransition { .. }) => 422, + BlazeDaemonError::Core(blaze_core::BlazeError::OperationInProgress { .. }) => 409, BlazeDaemonError::Core(blaze_core::BlazeError::BackendUnavailable { .. }) => 503, _ => 500, } diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index 4f654c85a9..05a5e8e175 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -17,6 +17,7 @@ mod failpoint; mod file_provider; mod metrics; mod request_body; +mod sandbox; mod spawner; mod state; diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs new file mode 100644 index 0000000000..2952f6f3ab --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Managed sandbox lifecycle and runtime ownership. + +mod manager; + +pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs new file mode 100644 index 0000000000..362daedd27 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -0,0 +1,1204 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable sandbox create, warm activation, destroy, and startup cleanup. + +use std::collections::{BTreeSet, HashMap}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, Weak}; +use std::time::Duration; + +use blaze_core::BlazeError; +use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, +}; +use blaze_core::policy::RuntimeDecision; +use blaze_core::pool::{PoolKey, PoolManager}; +use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::metrics::Metrics; +use crate::spawner::{DynBackendInstance, SpawnerRegistry}; + +const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); + +/// Inputs already parsed and policy-evaluated by the API. +#[derive(Debug, Clone)] +pub struct CreateSandbox { + /// Policy decision for this request. + pub decision: RuntimeDecision, + /// Image identity used by storage and warm-pool matching. + pub image_digest: String, + /// Concrete backend selected from the policy and daemon availability. + pub runtime_backend: BackendKind, + /// Executable selected during daemon startup. + pub binary_path: PathBuf, +} + +/// Result of one managed create request. +#[derive(Debug, Clone)] +pub struct CreateSandboxResult { + /// Persisted sandbox metadata. + pub instance: SandboxInstance, + /// Backend implementation that owns the runtime. + pub selected_backend: BackendKind, +} + +/// One startup cleanup failure. Other records continue to be reconciled. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReconcileFailure { + /// Sandbox whose cleanup remains incomplete. + pub instance_id: Uuid, + /// Actionable failure description. + pub error: String, +} + +/// Aggregate startup cleanup outcome. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ReconcileReport { + /// Number of records that were not clean terminal states. + pub attempted: usize, + /// Number moved to the terminal state. + pub completed: usize, + /// Records that remain recoverable. + pub failures: Vec, +} + +/// Owns durable lifecycle metadata and non-serializable runtime handles. +/// +/// The maps are shared with read-only and non-lifecycle API paths. All +/// create, warm activation, destroy, and restart cleanup mutations enter +/// through this type and are serialized by a per-sandbox async lock. +pub struct SandboxManager { + instances: Arc>>, + backend_instances: Arc>>, + operation_locks: Mutex>>>, + pool: Arc>, + spawners: Arc, + active_backend: BackendKind, + storage: Arc, + state_dir: PathBuf, + rootfs_size: u64, + mem_size: u64, + metrics: Arc, +} + +/// Construction inputs grouped to keep daemon wiring explicit. +pub struct SandboxManagerInit { + pub instances: HashMap, + pub pool: PoolManager, + pub spawners: SpawnerRegistry, + pub active_backend: BackendKind, + pub storage: Arc, + pub state_dir: PathBuf, + pub rootfs_size: u64, + pub mem_size: u64, +} + +/// Shared read-only resources retained by [`crate::state::ServerState`]. +pub struct SandboxManagerResources { + #[cfg(test)] + pub instances: Arc>>, + pub pool: Arc>, + pub metrics: Arc, +} + +impl SandboxManager { + /// Build a manager around state loaded from the durable state directory. + pub fn new(init: SandboxManagerInit) -> (Self, SandboxManagerResources) { + let SandboxManagerInit { + instances, + pool, + spawners, + active_backend, + storage, + state_dir, + rootfs_size, + mem_size, + } = init; + let instances = Arc::new(Mutex::new(instances)); + let backend_instances = Arc::new(Mutex::new(HashMap::new())); + let pool = Arc::new(Mutex::new(pool)); + let metrics = Arc::new(Metrics::new()); + let resources = SandboxManagerResources { + #[cfg(test)] + instances: instances.clone(), + pool: pool.clone(), + metrics: metrics.clone(), + }; + ( + Self { + instances, + backend_instances, + operation_locks: Mutex::new(HashMap::new()), + pool, + spawners: Arc::new(spawners), + active_backend, + storage, + state_dir, + rootfs_size, + mem_size, + metrics, + }, + resources, + ) + } + + /// Return the async operation lock that serializes one sandbox mutation. + pub fn operation_lock(&self, id: Uuid) -> Arc> { + match self.operation_locks.lock() { + Ok(mut locks) => operation_lock(&mut locks, id), + Err(poisoned) => operation_lock(&mut poisoned.into_inner(), id), + } + } + + /// Hold the per-sandbox mutation lock after confirming that the persisted + /// record is in `expected_state` and has no unfinished operation. + /// + /// Future lifecycle operations must retain the returned guard until their + /// final state and journal update have been persisted. + pub async fn lock_quiescent_state( + &self, + id: Uuid, + expected_state: SandboxState, + ) -> Result> { + let operation = self.operation_lock(id).lock_owned().await; + let instance = self.get(id)?; + if let Some(journal) = instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != expected_state { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected {}", + instance.state, expected_state + ))); + } + Ok(operation) + } + + #[cfg(test)] + pub fn backend_owner(&self, id: Uuid) -> Option { + match self.backend_instances.lock() { + Ok(instances) => instances.get(&id).cloned(), + Err(poisoned) => poisoned.into_inner().get(&id).cloned(), + } + } + + #[cfg(test)] + pub fn insert_backend_owner(&self, id: Uuid, backend: DynBackendInstance) -> Result<()> { + self.backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .insert(id, backend); + Ok(()) + } + + /// Return all persisted sandbox metadata. + pub fn list(&self) -> Result> { + Ok(self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .values() + .cloned() + .collect()) + } + + /// Return one persisted sandbox. + pub fn get(&self, id: Uuid) -> Result { + self.instances + .lock() + .map_err(|_| poisoned("instances"))? + .get(&id) + .cloned() + .ok_or_else(|| BlazeDaemonError::NotFound(format!("instance {id}"))) + } + + /// Return every sandbox for which lifecycle cleanup still owns resources. + /// + /// Shutdown uses this snapshot to start cleanup concurrently while all + /// mutations remain serialized by the manager's per-sandbox locks. + pub(crate) fn owned_instance_ids(&self) -> Result> { + let mut ids = self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .values() + .filter(|instance| !is_clean_terminal(instance)) + .map(|instance| instance.id) + .collect::>(); + ids.extend( + self.backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .keys() + .copied(), + ); + Ok(ids) + } + + /// Create a cold sandbox or activate a compatible warm runtime. + pub async fn create(&self, request: CreateSandbox) -> Result { + let pool_key = PoolKey::new( + request.runtime_backend, + request.decision.workload_class, + request.image_digest.clone(), + ); + if request.decision.pool_eligible { + if let Some(result) = self.activate_warm(&pool_key).await? { + self.metrics.inc(&self.metrics.pool_hits); + return Ok(result); + } + self.metrics.inc(&self.metrics.pool_misses); + } + + let mut instance = SandboxInstance::new( + request.runtime_backend, + request.decision.workload_class, + request.image_digest, + StartPath::Cold, + request.decision.policy_name.clone(), + ); + let operation_lock = self.operation_lock(instance.id); + let _operation = operation_lock.lock().await; + instance.transition(SandboxState::Creating)?; + instance.begin_operation(OperationKind::Create)?; + + // Publish the stable identity and create intent before allocation. + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "create {}: {error}", + instance.id + ))); + } + + let storage = match self + .storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: self.rootfs_size, + mem_size: self.mem_size, + }) + .await + { + Ok(storage) => storage, + Err(error) => { + let (source, residual) = error.into_parts(); + return Err(self.retain_failed_acquire(&mut instance, residual, source.into())); + } + }; + crate::failpoint::pause("create-after-storage-acquire").await; + + let run_dir = self.state_dir.join(instance.id.to_string()); + let spawner = match self.spawners.get(self.active_backend) { + Some(spawner) => spawner, + None => { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(format!( + "active backend {} has no registered spawner", + self.active_backend + )), + ) + .await); + } + }; + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, false, error.into()) + .await); + } + + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance.persist(&self.state_dir) { + instance.backend_ownership = BackendOwnership::NotStarted; + return Err(self + .cleanup_failed_create(&mut instance, storage, None, false, error.into()) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + instance.backend_ownership = BackendOwnership::NotStarted; + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + false, + BlazeDaemonError::Internal(error), + ) + .await); + } + + let spawn = match crate::failpoint::backend("create-spawn") { + Ok(()) => { + spawner + .spawn(SpawnRequest { + instance_id: instance.id, + run_dir, + binary_path: request.binary_path, + storage: storage.clone(), + backend: request.decision.backend, + vm: request.decision.vm, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let actual_backend = match spawn { + Ok(backend_instance) => { + instance.backend_ownership = BackendOwnership::Running; + let actual_backend = backend_instance.backend(); + let mut backend_instance = Some(backend_instance); + let registered = match self.backend_instances.lock() { + Ok(mut instances) => { + instances.insert( + instance.id, + backend_instance + .take() + .expect("backend instance is present"), + ); + true + } + Err(_) => false, + }; + if !registered { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + backend_instance, + false, + BlazeDaemonError::Internal( + "backend_instances lock poisoned".to_string(), + ), + ) + .await); + } + actual_backend + } + Err(error) => { + let (source, backend) = error.into_parts(); + instance.backend_ownership = if backend.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::Stopped + }; + return Err(self + .cleanup_failed_create(&mut instance, storage, backend, false, source.into()) + .await); + } + }; + + if let Err(error) = instance.transition(SandboxState::Running) { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, true, error.into()) + .await); + } + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("create-state-commit") + .and_then(|_| instance.persist(&self.state_dir).map_err(Into::into)) + { + return Err(self + .cleanup_failed_create(&mut instance, storage, None, true, error) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + None, + true, + BlazeDaemonError::Internal(error), + ) + .await); + } + self.metrics.inc(&self.metrics.instances_created); + Ok(CreateSandboxResult { + instance, + selected_backend: actual_backend, + }) + } + + async fn activate_warm(&self, key: &PoolKey) -> Result> { + let candidate = self.pool.lock().map_err(|_| poisoned("pool"))?.lookup(key); + let Some(id) = candidate else { + return Ok(None); + }; + let operation_lock = self.operation_lock(id); + let _operation = operation_lock.lock().await; + + let instance = self + .instances + .lock() + .map_err(|_| poisoned("instances"))? + .get(&id) + .cloned(); + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + + let invalid_reason = match (&instance, &backend) { + (None, _) => Some("lifecycle metadata is missing".to_string()), + (_, None) => Some("backend owner is missing".to_string()), + (Some(instance), Some(_)) + if instance.state != SandboxState::Warm || instance.operation.is_some() => + { + Some(format!("lifecycle state is {}", instance.state)) + } + (Some(instance), Some(backend)) if backend.backend() != instance.backend => { + Some(format!( + "backend owner is {}, metadata is {}", + backend.backend(), + instance.backend + )) + } + (_, Some(backend)) => match backend.try_wait().await { + Ok(None) => None, + Ok(Some(status)) => Some(format!("backend exited: {status:?}")), + Err(error) => Some(format!("backend liveness check failed: {error}")), + }, + }; + if let Some(reason) = invalid_reason { + self.quarantine_warm(key, id, instance, backend, &reason) + .await; + return Ok(None); + } + + let original = instance.expect("validated warm metadata"); + match self.storage.reconstruct(&id.to_string()).await { + Ok(_) => {} + Err(error @ BlazeError::StorageIncomplete { .. }) => { + self.quarantine_warm( + key, + id, + Some(original), + backend, + &format!("storage validation failed: {error}"), + ) + .await; + return Ok(None); + } + Err(error) => { + return Err(self.restore_warm_claim(key, original, error.into())); + } + } + + let mut activating = original.clone(); + if let Err(error) = activating.begin_operation(OperationKind::Create) { + return Err(self.restore_warm_claim(key, original, error.into())); + } + if let Err(error) = crate::failpoint::state("warm-intent-state-commit").and_then(|_| { + activating + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + return Err(self.restore_warm_claim(key, original, error)); + } + if let Some(error) = self.retain_instance(activating.clone()) { + return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); + } + + crate::failpoint::pause("warm-before-state-commit").await; + let selected_backend = backend.expect("validated warm backend").backend(); + if let Err(error) = activating + .transition(SandboxState::Creating) + .and_then(|_| activating.transition(SandboxState::Running)) + { + return Err(self.restore_warm_claim(key, original, error.into())); + } + activating.finish_operation(); + if let Err(error) = crate::failpoint::state("warm-final-state-commit").and_then(|_| { + activating + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + return Err(self.restore_warm_claim(key, original, error)); + } + if let Some(error) = self.retain_instance(activating.clone()) { + return Err(self.restore_warm_claim(key, original, BlazeDaemonError::Internal(error))); + } + Ok(Some(CreateSandboxResult { + instance: activating, + selected_backend, + })) + } + + fn restore_warm_claim( + &self, + key: &PoolKey, + instance: SandboxInstance, + cause: BlazeDaemonError, + ) -> BlazeDaemonError { + let id = instance.id; + let mut errors = Vec::new(); + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("restore warm state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance) { + errors.push(error); + } + match self.pool.lock() { + Ok(mut pool) => pool.restore_lookup(key.clone(), id), + Err(poisoned) => { + poisoned.into_inner().restore_lookup(key.clone(), id); + errors.push("pool lock poisoned while restoring warm claim".to_string()); + } + } + let details = if errors.is_empty() { + "warm claim restored for retry".to_string() + } else { + format!("warm claim restored with errors: {}", errors.join("; ")) + }; + BlazeDaemonError::RecoveryRequired(format!("{cause}; instance {id}: {details}")) + } + + async fn quarantine_warm( + &self, + key: &PoolKey, + id: Uuid, + mut instance: Option, + backend: Option, + reason: &str, + ) { + match self.pool.lock() { + Ok(mut pool) => pool.quarantine(key, id), + Err(poisoned) => poisoned.into_inner().quarantine(key, id), + } + tracing::warn!(instance = %id, reason, "warm instance validation failed"); + + let Some(metadata) = instance.as_mut() else { + if let Some(backend) = backend.as_ref() + && let Err(error) = backend.kill().await + { + tracing::error!( + instance = %id, + %error, + "quarantined backend cleanup failed" + ); + } + tracing::error!( + instance = %id, + "quarantined lifecycle metadata missing; retaining storage" + ); + return; + }; + metadata.begin_destroy_recovery(); + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantine intent commit failed"); + return; + } + if let Some(error) = self.retain_instance(metadata.clone()) { + tracing::error!(instance = %id, %error, "quarantine intent retention failed"); + return; + } + + let backend_stopped = match backend.as_ref() { + Some(backend) => match backend.kill().await { + Ok(()) => true, + Err(error) => { + tracing::error!(instance = %id, %error, "quarantined backend cleanup failed"); + false + } + }, + None if matches!( + metadata.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) => + { + true + } + None => match self.spawners.get(metadata.backend) { + Some(spawner) => match spawner + .cleanup_orphan(id, &self.state_dir.join(id.to_string())) + .await + { + Ok(()) => true, + Err(error) => { + tracing::error!( + instance = %id, + %error, + "quarantined orphan cleanup failed" + ); + false + } + }, + None => { + tracing::error!( + instance = %id, + backend = %metadata.backend, + "quarantined backend has no recovery spawner" + ); + false + } + }, + }; + if !backend_stopped { + let _ = self.mark_recovery(id); + return; + } + + metadata.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantined stop state commit failed"); + let _ = self.mark_recovery(id); + return; + } + if let Some(error) = self.retain_instance(metadata.clone()) { + tracing::error!(instance = %id, %error, "quarantined stop state retention failed"); + let _ = self.mark_recovery(id); + return; + } + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { + tracing::error!(instance = %id, %error, "quarantined storage cleanup failed"); + let _ = self.mark_recovery(id); + return; + } + + if metadata.state != SandboxState::Destroyed + && let Err(error) = metadata.transition(SandboxState::Destroyed) + { + tracing::error!(instance = %id, %error, "quarantined lifecycle cleanup failed"); + let _ = self.mark_recovery(id); + return; + } + metadata.finish_operation(); + if let Err(error) = metadata.persist(&self.state_dir) { + tracing::error!(instance = %id, %error, "quarantined state commit failed"); + let _ = self.mark_recovery(id); + return; + } + let _ = self.retain_instance(metadata.clone()); + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.remove(&id); + } + Err(poisoned) => { + poisoned.into_inner().remove(&id); + } + } + } + + /// Idempotently destroy one sandbox and its owned runtime resources. + pub async fn destroy(&self, id: Uuid) -> Result { + let operation_lock = self.operation_lock(id); + let _operation = operation_lock.lock().await; + self.destroy_locked(id).await + } + + async fn destroy_locked(&self, id: Uuid) -> Result { + let mut original = self.get(id)?; + let has_retained_backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .contains_key(&id); + if is_clean_terminal(&original) && !has_retained_backend { + return Ok(false); + } + + if original.operation.as_ref().map(|operation| operation.kind) + != Some(OperationKind::Destroy) + { + original.begin_destroy_recovery(); + } + if let Err(error) = crate::failpoint::state("destroy-intent-state-commit").and_then(|_| { + original + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: intent persistence failed: {error}; resources retained" + ))); + } + if let Some(error) = self.retain_instance(original.clone()) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: {error}; resources retained" + ))); + } + + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + let stop_result = match crate::failpoint::backend("destroy-kill") { + Ok(()) => { + if let Some(backend) = backend.as_ref() { + backend.kill().await + } else if matches!( + original.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) { + Ok(()) + } else { + match self.spawners.get(original.backend) { + Some(spawner) => { + spawner + .cleanup_orphan(id, &self.state_dir.join(id.to_string())) + .await + } + None => Err(BlazeError::BackendError { + msg: format!( + "no recovery spawner registered for persisted backend {}", + original.backend + ), + }), + } + } + } + Err(error) => Err(error), + }; + if let Err(error) = stop_result { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend termination failed: {error}; owner and storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + original.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = crate::failpoint::state("destroy-stop-state-commit").and_then(|_| { + original + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let recovery = self.mark_instance_recovery(original.clone()).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but stop state persistence failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(error) = self.retain_instance(original.clone()) { + let _ = self.mark_recovery(id); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but lifecycle retention failed: {error}; \ + storage retained" + ))); + } + + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but storage release failed: {error}; \ + lifecycle retained for retry{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + let mut destroyed = original; + if destroyed.state != SandboxState::Destroyed { + destroyed.transition(SandboxState::Destroyed)?; + } + destroyed.finish_operation(); + if let Err(error) = crate::failpoint::state("destroy-final-state-commit").and_then(|_| { + destroyed + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + }) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: resources released but final state persistence failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(error) = self.retain_instance(destroyed) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: resources released but {error}" + ))); + } + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.remove(&id); + } + Err(poisoned) => { + poisoned.into_inner().remove(&id); + } + } + self.metrics.inc(&self.metrics.instances_destroyed); + Ok(true) + } + + /// Reconcile every record that is not cleanly terminal without aborting + /// on one failure. + pub async fn reconcile_startup(&self) -> ReconcileReport { + self.cleanup_owned_instances().await + } + + /// Release every lifecycle record and retained backend owner, continuing + /// after individual failures. + pub async fn cleanup_owned_instances(&self) -> ReconcileReport { + self.cleanup_owned_instances_with_timeout(INSTANCE_CLEANUP_TIMEOUT) + .await + } + + /// Release owned instances with a caller-supplied per-instance deadline. + pub async fn cleanup_owned_instances_with_timeout( + &self, + item_timeout: Duration, + ) -> ReconcileReport { + let mut ids = match self.instances.lock() { + Ok(instances) => instances + .values() + .filter(|instance| !is_clean_terminal(instance)) + .map(|instance| instance.id) + .collect::>(), + Err(poisoned) => poisoned + .into_inner() + .values() + .filter(|instance| !is_clean_terminal(instance)) + .map(|instance| instance.id) + .collect::>(), + }; + match self.backend_instances.lock() { + Ok(instances) => ids.extend(instances.keys().copied()), + Err(poisoned) => ids.extend(poisoned.into_inner().keys().copied()), + } + self.cleanup_instances_with_timeout(ids, item_timeout).await + } + + async fn cleanup_instances_with_timeout( + &self, + ids: BTreeSet, + item_timeout: Duration, + ) -> ReconcileReport { + let mut report = ReconcileReport { + attempted: ids.len(), + ..ReconcileReport::default() + }; + for id in ids { + let operation_lock = self.operation_lock(id); + let deadline = tokio::time::Instant::now() + item_timeout; + let _operation = match tokio::time::timeout_at(deadline, operation_lock.lock()).await { + Ok(operation) => operation, + Err(_) => { + report.failures.push(ReconcileFailure { + instance_id: id, + error: format!( + "instance cleanup exceeded {} ms per-record deadline while waiting \ + for ownership", + item_timeout.as_millis() + ), + }); + continue; + } + }; + match tokio::time::timeout_at(deadline, self.destroy_locked(id)).await { + Ok(Ok(_)) => report.completed += 1, + Err(_) => { + let error = format!( + "instance cleanup exceeded {} ms per-record deadline", + item_timeout.as_millis() + ); + let recovery = self.mark_recovery(id).err(); + report.failures.push(ReconcileFailure { + instance_id: id, + error: match recovery { + Some(recovery) => { + format!("{error}; recovery state persistence failed: {recovery}") + } + None => error, + }, + }); + } + Ok(Err(error)) => { + let recovery = self.mark_recovery(id).err(); + report.failures.push(ReconcileFailure { + instance_id: id, + error: match recovery { + Some(recovery) => { + format!("{error}; recovery state persistence failed: {recovery}") + } + None => error.to_string(), + }, + }); + } + } + } + report + } + + async fn cleanup_failed_create( + &self, + instance: &mut SandboxInstance, + storage: StorageSlot, + backend: Option, + registered: bool, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + if instance.operation.is_none() + && let Err(error) = instance.begin_operation(OperationKind::Create) + { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {}: cannot retain create journal: {error}", + instance.id + )); + } + let mut cleanup_errors = Vec::new(); + let backend = if registered { + match self.backend_instances.lock() { + Ok(mut instances) => instances.remove(&instance.id), + Err(poisoned) => poisoned.into_inner().remove(&instance.id), + } + } else { + backend + }; + let mut backend_stopped = matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ); + if registered && backend.is_none() { + backend_stopped = false; + cleanup_errors.push("registered backend owner is missing".to_string()); + } + if let Some(backend) = backend.as_ref() { + match backend.kill().await { + Ok(()) => { + backend_stopped = true; + instance.backend_ownership = BackendOwnership::Stopped; + } + Err(error) => { + backend_stopped = false; + cleanup_errors.push(format!("backend termination failed: {error}")); + } + } + } + + let mut storage_released = false; + if backend_stopped { + match self.storage.release(storage).await { + Ok(()) => storage_released = true, + Err(error) => cleanup_errors.push(format!("storage release failed: {error}")), + } + } else { + cleanup_errors.push("storage retained until backend termination succeeds".to_string()); + } + + if backend_stopped && storage_released { + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Destroyed) { + cleanup_errors.push(format!("lifecycle update failed: {error}")); + } else { + instance.finish_operation(); + } + if let Err(error) = instance.persist(&self.state_dir) { + cleanup_errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + cleanup_errors.push(error); + } + if cleanup_errors.is_empty() { + self.metrics.inc(&self.metrics.instances_destroyed); + return original; + } + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; cleanup completed but {}", + cleanup_errors.join("; ") + )); + } + + if let Some(backend) = backend + && let Some(error) = self.retain_backend(instance.id, backend) + { + cleanup_errors.push(error); + } + if instance.state != SandboxState::RecoveryRequired + && let Err(error) = instance.transition(SandboxState::RecoveryRequired) + { + cleanup_errors.push(format!("recovery state update failed: {error}")); + } + if let Err(error) = instance.persist(&self.state_dir) { + cleanup_errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + cleanup_errors.push(error); + } + BlazeDaemonError::RecoveryRequired(format!( + "{original}; cleanup incomplete: {}", + cleanup_errors.join("; ") + )) + } + + fn retain_failed_acquire( + &self, + instance: &mut SandboxInstance, + residual: Option, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + if residual.is_some() { + let mut errors = Vec::new(); + if instance.state != SandboxState::RecoveryRequired + && let Err(error) = instance.transition(SandboxState::RecoveryRequired) + { + errors.push(format!("recovery state update failed: {error}")); + } + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + errors.push(error); + } + let suffix = if errors.is_empty() { + "residual storage retained for destroy retry".to_string() + } else { + format!( + "residual storage retained with recovery errors: {}", + errors.join("; ") + ) + }; + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {}: {suffix}", + instance.id + )); + } + + let mut errors = Vec::new(); + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Destroyed) { + errors.push(format!("lifecycle update failed: {error}")); + } else { + instance.finish_operation(); + } + if let Err(error) = instance.persist(&self.state_dir) { + errors.push(format!("state persistence failed: {error}")); + } + if let Some(error) = self.retain_instance(instance.clone()) { + errors.push(error); + } + if errors.is_empty() { + original + } else { + BlazeDaemonError::RecoveryRequired(format!( + "{original}; acquire rollback completed but {}", + errors.join("; ") + )) + } + } + + fn mark_recovery(&self, id: Uuid) -> Result<()> { + self.mark_instance_recovery(self.get(id)?) + } + + fn mark_instance_recovery(&self, mut instance: SandboxInstance) -> Result<()> { + if instance.state == SandboxState::Destroyed { + if instance.operation.as_ref().map(|operation| operation.kind) + != Some(OperationKind::Destroy) + { + instance.begin_destroy_recovery(); + } + } else if instance.state != SandboxState::RecoveryRequired { + instance.transition(SandboxState::RecoveryRequired)?; + } + let persist = instance.persist(&self.state_dir); + let retained = self.retain_instance(instance); + match (persist, retained) { + (Ok(()), None) => Ok(()), + (Err(error), None) => Err(error.into()), + (Ok(()), Some(error)) => Err(BlazeDaemonError::Internal(error)), + (Err(persist), Some(retain)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "recovery state persistence failed: {persist}; {retain}" + ))), + } + } + + fn retain_backend(&self, id: Uuid, backend: DynBackendInstance) -> Option { + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.insert(id, backend); + None + } + Err(poisoned) => { + poisoned.into_inner().insert(id, backend); + Some("backend owner retained in poisoned runtime map".to_string()) + } + } + } + + fn retain_instance(&self, instance: SandboxInstance) -> Option { + match self.instances.lock() { + Ok(mut instances) => { + instances.insert(instance.id, instance); + None + } + Err(poisoned) => { + poisoned.into_inner().insert(instance.id, instance); + Some("instance state retained in poisoned lifecycle map".to_string()) + } + } + } +} + +fn poisoned(name: &str) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("{name} lock poisoned")) +} + +fn is_clean_terminal(instance: &SandboxInstance) -> bool { + instance.state == SandboxState::Destroyed + && instance.operation.is_none() + && matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) +} + +fn operation_lock( + locks: &mut HashMap>>, + id: Uuid, +) -> Arc> { + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&id).and_then(Weak::upgrade) { + return lock; + } + + let lock = Arc::new(AsyncMutex::new(())); + locks.insert(id, Arc::downgrade(&lock)); + lock +} + +#[cfg(test)] +mod operation_lock_tests { + use super::*; + + #[test] + fn live_operation_locks_are_reused_and_dead_entries_are_pruned() { + let mut locks = HashMap::new(); + let id = Uuid::new_v4(); + let first = operation_lock(&mut locks, id); + let second = operation_lock(&mut locks, id); + + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(locks.len(), 1); + + drop(first); + drop(second); + for _ in 0..256 { + drop(operation_lock(&mut locks, Uuid::new_v4())); + } + + assert_eq!(locks.len(), 1); + } +} diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index 70192aabf8..be8b5cf5b1 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -1,13 +1,15 @@ // SPDX-License-Identifier: Apache-2.0 -//! Daemon-wide shared state: configuration, policy engine, pool, template -//! and hook registries, and the in-memory instance map. All API handlers -//! receive an [`Arc`] and acquire the relevant `Mutex<...>` -//! lock just long enough to read or mutate the piece they need — locks -//! are never held across `.await` boundaries. +//! Daemon-wide shared state: configuration, policy engine, pool, template and +//! hook registries, plus the sandbox manager. API paths that change runtime +//! ownership enter through the manager so its per-instance lock spans every +//! asynchronous resource mutation. use std::collections::HashMap; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex, Weak}; +use std::path::Path; +use std::sync::{Arc, Mutex}; + +#[cfg(test)] +use std::path::PathBuf; use blaze_core::backend::BackendKind; use blaze_core::config::DaemonConfig; @@ -17,32 +19,32 @@ use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; use blaze_core::storage::StorageProvider; use blaze_core::template::TemplateRegistry; -use tokio::sync::Mutex as AsyncMutex; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::metrics::Metrics; -use crate::spawner::{DynBackendInstance, DynSpawner, SpawnerRegistry}; +use crate::sandbox::{SandboxManager, SandboxManagerInit}; +use crate::spawner::SpawnerRegistry; /// All daemon mutable state. Cloning is via `Arc` (see the `state.clone()` /// idiom in `daemon.rs`); the struct itself is never `Clone`. pub struct ServerState { pub config: Mutex, pub policy: Mutex, - pub pool: Mutex, + pub pool: Arc>, pub template: Mutex, pub hook: Mutex, - pub instances: Mutex>, - pub backend_instances: Mutex>, - operation_locks: Mutex>>>, - pub spawners: SpawnerRegistry, + #[cfg(test)] + pub instances: Arc>>, + pub manager: SandboxManager, /// The backend kind that `build_spawner` actually probed and selected. /// API handlers use this to constrain availability to the single active /// backend rather than reporting all configured binaries. pub active_backend: BackendKind, pub storage: Arc, + #[cfg(test)] pub state_dir: PathBuf, - pub metrics: Metrics, + pub metrics: Arc, } impl ServerState { @@ -61,50 +63,39 @@ impl ServerState { ) -> Result { let state_dir = config.daemon.state_dir.clone(); let instances = scan_state_dir(&state_dir)?; + let (manager, resources) = SandboxManager::new(SandboxManagerInit { + instances, + pool, + spawners, + active_backend, + storage: storage.clone(), + state_dir: state_dir.clone(), + rootfs_size: config.storage.rootfs_size, + mem_size: config.storage.mem_size, + }); Ok(Self { config: Mutex::new(config), policy: Mutex::new(policy), - pool: Mutex::new(pool), + pool: resources.pool, template: Mutex::new(template), hook: Mutex::new(hook), - instances: Mutex::new(instances), - backend_instances: Mutex::new(HashMap::new()), - operation_locks: Mutex::new(HashMap::new()), - spawners, + #[cfg(test)] + instances: resources.instances, + manager, active_backend, storage, + #[cfg(test)] state_dir, - metrics: Metrics::new(), + metrics: resources.metrics, }) } /// Return the async operation lock that serializes one sandbox mutation. - pub fn operation_lock(&self, id: Uuid) -> Arc> { - match self.operation_locks.lock() { - Ok(mut locks) => operation_lock(&mut locks, id), - Err(poisoned) => operation_lock(&mut poisoned.into_inner(), id), - } - } - - /// Return the implementation responsible for a persisted backend kind. - pub fn spawner_for(&self, kind: BackendKind) -> Option { - self.spawners.get(kind) - } -} - -fn operation_lock( - locks: &mut HashMap>>, - id: Uuid, -) -> Arc> { - locks.retain(|_, lock| lock.strong_count() > 0); - if let Some(lock) = locks.get(&id).and_then(Weak::upgrade) { - return lock; + #[cfg(test)] + pub fn operation_lock(&self, id: Uuid) -> Arc> { + self.manager.operation_lock(id) } - - let lock = Arc::new(AsyncMutex::new(())); - locks.insert(id, Arc::downgrade(&lock)); - lock } /// Walk `{state_dir}//state.json` and rebuild the instance map. @@ -193,23 +184,4 @@ mod tests { && message.contains(&directory_id.to_string()) )); } - - #[test] - fn operation_locks_reuse_live_entries_and_prune_dead_ones() { - let mut locks = HashMap::new(); - let id = Uuid::new_v4(); - let first = operation_lock(&mut locks, id); - let second = operation_lock(&mut locks, id); - - assert!(Arc::ptr_eq(&first, &second)); - assert_eq!(locks.len(), 1); - - drop(first); - drop(second); - for _ in 0..256 { - drop(operation_lock(&mut locks, Uuid::new_v4())); - } - - assert_eq!(locks.len(), 1); - } } From 9f11db38b499d82445319535f5142147509e1e0a Mon Sep 17 00:00:00 2001 From: Weisson Date: Wed, 29 Jul 2026 18:23:27 +0800 Subject: [PATCH 17/40] feat(blaze): expose sandbox lifecycle routes Lifecycle management was available only through the instance route family, and destroy required the older POST action form. Callers could not treat a sandbox as a conventional collection and item resource. Add sandbox collection and item aliases plus DELETE forms for sandbox and instance resources. Every route delegates to the same managed create, list, inspect, or destroy handler. This enables callers to create, enumerate, inspect, and delete sandboxes through `/v1/sandboxes` without introducing a second lifecycle implementation. Existing instance routes, checkpoint and reset actions, and response formats remain available. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blazed/src/api.rs | 125 ++++++++++++++++++++++++++++- 1 file changed, 121 insertions(+), 4 deletions(-) diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index e753ab5cd8..54753d1dfd 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -79,12 +79,18 @@ async fn dispatch( match (m, parts.as_slice()) { ("GET", ["v1", "health"]) => health(state), - ("GET", ["v1", "instances"]) => list_instances(state), - ("POST", ["v1", "instances"]) => create_instance(state, &body).await, - ("GET", ["v1", "instances", id]) => get_instance(state, id), + ("GET", ["v1", "instances"]) | ("GET", ["v1", "sandboxes"]) => list_instances(state), + ("POST", ["v1", "instances"]) | ("POST", ["v1", "sandboxes"]) => { + create_instance(state, &body).await + } + ("GET", ["v1", "instances", id]) | ("GET", ["v1", "sandboxes", id]) => { + get_instance(state, id) + } ("POST", ["v1", "instances", id, "checkpoint"]) => checkpoint(state, id).await, ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, - ("POST", ["v1", "instances", id, "destroy"]) => destroy_instance(state, id).await, + ("DELETE", ["v1", "instances", id]) + | ("DELETE", ["v1", "sandboxes", id]) + | ("POST", ["v1", "instances", id, "destroy"]) => destroy_instance(state, id).await, ("GET", ["v1", "pools"]) => list_pools(state), ("GET", ["v1", "pools", backend, class]) => pool_status(state, backend, class), ("POST", ["v1", "pools", backend, class, "drain"]) => drain_pool(state, backend, class), @@ -828,6 +834,26 @@ mod tests { .expect("created json") } + async fn dispatched_json( + state: &Arc, + method: Method, + path: &str, + body: Vec, + ) -> (StatusCode, serde_json::Value) { + let response = dispatch(&method, path, "", body, state) + .await + .expect("dispatch"); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value = serde_json::from_slice(&body).expect("response json"); + (status, value) + } + struct TransientReconstructStorage { inner: FileStorageProvider, fail_reconstruct: AtomicBool, @@ -1349,6 +1375,97 @@ mod tests { (state, kill_count, orphan_cleanup_count, release_count) } + #[tokio::test] + async fn sandbox_collection_and_item_routes_match_instance_routes() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let (status, created) = + dispatched_json(&state, Method::POST, "/v1/sandboxes", test_request()).await; + assert_eq!(status, StatusCode::CREATED); + assert_eq!(created["instance"]["state"], "running"); + let id = created["instance"]["id"].as_str().expect("instance id"); + + let (_, sandboxes) = + dispatched_json(&state, Method::GET, "/v1/sandboxes", Vec::new()).await; + let (_, instances) = + dispatched_json(&state, Method::GET, "/v1/instances", Vec::new()).await; + assert_eq!(sandboxes, instances); + + let (_, sandbox) = dispatched_json( + &state, + Method::GET, + &format!("/v1/sandboxes/{id}"), + Vec::new(), + ) + .await; + let (_, instance) = dispatched_json( + &state, + Method::GET, + &format!("/v1/instances/{id}"), + Vec::new(), + ) + .await; + assert_eq!(sandbox, instance); + } + + #[tokio::test] + async fn destroy_route_forms_share_managed_cleanup() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let mut ids = Vec::new(); + for _ in 0..3 { + let created = created_json(&state, &test_request()).await; + ids.push( + Uuid::parse_str(created["instance"]["id"].as_str().expect("instance id")) + .expect("uuid"), + ); + } + let routes = [ + (Method::DELETE, format!("/v1/sandboxes/{}", ids[0]), ids[0]), + (Method::DELETE, format!("/v1/instances/{}", ids[1]), ids[1]), + ( + Method::POST, + format!("/v1/instances/{}/destroy", ids[2]), + ids[2], + ), + ]; + + for (method, path, id) in routes { + let (status, response) = dispatched_json(&state, method, &path, Vec::new()).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(response["destroyed"], true); + assert_eq!(response["instance_id"], id.to_string()); + assert_eq!( + state.manager.get(id).expect("destroyed state").state, + SandboxState::Destroyed + ); + } + } + /// When multiple backend binaries exist on disk but the daemon probed /// Firecracker at boot, only Firecracker should be reported available /// and selected — even if policy prioritizes bubblewrap higher. From b4c973ebe73c4f9c49712e15f18cb201887ec92c Mon Sep 17 00:00:00 2001 From: Weisson Date: Wed, 29 Jul 2026 18:26:15 +0800 Subject: [PATCH 18/40] docs(blaze): describe managed sandboxes The API reference listed only instance-named routes and did not explain what happens when create, destroy, startup recovery, or daemon shutdown stops partway through resource cleanup. Document sandbox and compatible instance routes, the operation journal, RecoveryRequired state, per-record startup reconciliation, and ordered graceful shutdown cleanup. The English and Chinese examples now use the sandbox collection while retaining all compatible route forms. This enables operators and reviewers to distinguish implemented lifecycle behavior from reserved interfaces. Checkpoint and reset are documented as validation-only routes that return `501` without changing runtime or persisted state. The documented limits remain explicit: interrupted create is cleaned up rather than resumed, existing backend processes are not adopted after restart, and failed recovery has no background retry loop. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/AGENTS.md | 6 +++++- src/blaze/README.md | 42 ++++++++++++++++++++++++++++++++++++------ src/blaze/README_zh.md | 37 +++++++++++++++++++++++++++++++------ 3 files changed, 72 insertions(+), 13 deletions(-) diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index c34acdec25..7580ddefe6 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -29,7 +29,11 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw - **Daemon-only API model**: No CLI client for sandbox operations. All instance/pool/template management is done via HTTP endpoints on UDS (`/run/blaze/api.sock`) or TCP (`:14159`). The CLI subcommands (`daemon start`, `daemon reload`, `daemon doctor`) only manage daemon lifecycle. - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. -- **Lifecycle state machine**: 8 states (Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed). State transitions are enforced by `blaze_core::lifecycle`. Do not bypass via direct field mutation. +- **Lifecycle state machine**: The persisted model contains 9 states. Current + managed routes implement create, destroy, and RecoveryRequired cleanup. + Checkpoint and reset routes are reserved and return `501` without advancing + state. Transitions are enforced by `blaze_core::lifecycle`; do not bypass + them through direct field mutation. - **MockSpawner fallback**: When the configured backend binary is missing or fails `probe()`, the daemon auto-downgrades to `MockSpawner` with a warning. This keeps API/integration tests functional without a real backend. ## Adding a New Backend diff --git a/src/blaze/README.md b/src/blaze/README.md index a00f9f81e1..e8988d66f5 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -13,7 +13,8 @@ Designed as the per-host agent for E2B-style orchestrator platforms. - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **Policy-driven backend selection** — workload class → backend priority list -- **Lifecycle state machine** — 8 states (Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed) +- **Lifecycle state machine** — 9 states: Pending, Creating, Running, Paused, + Checkpointed, RecoveryRequired, Reset, Warm, and Destroyed - **Warm pool management** — pre-warmed instances with TTL-based GC - **Template registry** — in-memory template tracking with idle eviction - **Kernel hook registry** — state tracking for pre/post hooks @@ -38,7 +39,7 @@ sudo ./target/release/blazed daemon start --config examples/config.toml curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # Create a sandbox -curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/instances \ +curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' ``` @@ -112,12 +113,17 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | Method | Path | Description | |--------|------|-------------| | GET | `/v1/health` | Health check | -| GET | `/v1/instances` | List all instances | -| POST | `/v1/instances` | Create a new sandbox instance | -| GET | `/v1/instances/{id}` | Get instance details | +| GET | `/v1/sandboxes` | List all sandboxes | +| POST | `/v1/sandboxes` | Create a sandbox | +| GET | `/v1/sandboxes/{id}` | Get sandbox details | +| DELETE | `/v1/sandboxes/{id}` | Destroy a sandbox | +| GET | `/v1/instances` | Alias for listing sandboxes | +| POST | `/v1/instances` | Alias for creating a sandbox | +| GET | `/v1/instances/{id}` | Alias for sandbox details | +| DELETE | `/v1/instances/{id}` | Alias for destroying a sandbox | +| POST | `/v1/instances/{id}/destroy` | Compatible destroy action | | POST | `/v1/instances/{id}/checkpoint` | Reserved; returns `501` until backend and storage capture is implemented | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | -| POST | `/v1/instances/{id}/destroy` | Destroy an instance | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | | POST | `/v1/pools/{backend}/{class}/drain` | Drain a pool | @@ -130,6 +136,30 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | GET | `/v1/metrics` | Prometheus metrics | | POST | `/v1/admin/reload` | Hot-reload policies | +### Managed lifecycle and recovery + +Create and destroy record their operation before changing storage or backend +resources. A successful create finishes in `Running`; a successful destroy +finishes in `Destroyed`. If compensation cannot release every owned resource, +the sandbox remains visible as `RecoveryRequired` so destroy can be retried. + +At startup, the daemon reconciles each non-terminal sandbox independently. +Failure to clean up one sandbox does not prevent the remaining records from +being processed or the API from starting. + +During graceful shutdown, the daemon first stops accepting work and drains +accepted connections. It then attempts bounded cleanup for every persisted +record and retained backend owner. One cleanup failure does not skip the +remaining sandboxes, and all unresolved records are reported. + +The operation journal records the operation and start time, not completion of +each resource step. An interrupted create is cleaned up rather than resumed, +and an existing backend process is not adopted after restart. Failed recovery +does not run in a background retry loop. Checkpoint and reset validate that the +sandbox is running with no active lifecycle operation, then return `501` +without changing runtime or persisted state. Their backend and storage +operations are not implemented here. + #### Health Check `GET /v1/health` returns daemon status including storage pool readiness: diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index 69fc2a0bd7..ff96fa68e7 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -12,7 +12,8 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **策略驱动后端选择** — workload class → 后端优先级列表 -- **生命周期状态机** — 8 种状态（Pending → Creating → Running → Paused → Checkpointed → Reset → Warm → Destroyed） +- **生命周期状态机** — 9 种状态：Pending、Creating、Running、Paused、 + Checkpointed、RecoveryRequired、Reset、Warm 和 Destroyed - **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC - **模板注册表** — 内存中模板追踪，支持空闲驱逐 - **内核 hook 注册** — 前/后置 hook 状态追踪 @@ -37,7 +38,7 @@ sudo ./target/release/blazed daemon start --config examples/config.toml curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # 创建 sandbox -curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/instances \ +curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' ``` @@ -110,12 +111,17 @@ images_dir = "/var/lib/blaze/images" | 方法 | 路径 | 说明 | |--------|------|-------------| | GET | `/v1/health` | 健康检查 | -| GET | `/v1/instances` | 列出所有实例 | -| POST | `/v1/instances` | 创建新 sandbox 实例 | -| GET | `/v1/instances/{id}` | 获取实例详情 | +| GET | `/v1/sandboxes` | 列出所有 sandbox | +| POST | `/v1/sandboxes` | 创建 sandbox | +| GET | `/v1/sandboxes/{id}` | 获取 sandbox 详情 | +| DELETE | `/v1/sandboxes/{id}` | 销毁 sandbox | +| GET | `/v1/instances` | 列出 sandbox 的兼容入口 | +| POST | `/v1/instances` | 创建 sandbox 的兼容入口 | +| GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | +| DELETE | `/v1/instances/{id}` | 销毁 sandbox 的兼容入口 | +| POST | `/v1/instances/{id}/destroy` | 保留的销毁 action | | POST | `/v1/instances/{id}/checkpoint` | 预留接口；后端和存储快照实现前返回 `501` | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | -| POST | `/v1/instances/{id}/destroy` | 销毁实例 | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | | POST | `/v1/pools/{backend}/{class}/drain` | 排空 pool | @@ -128,6 +134,25 @@ images_dir = "/var/lib/blaze/images" | GET | `/v1/metrics` | Prometheus 指标 | | POST | `/v1/admin/reload` | 热加载策略 | +### 生命周期管理与恢复 + +创建和销毁会在修改存储或后端资源之前记录当前操作。创建成功后状态为 +`Running`，销毁成功后状态为 `Destroyed`。如果失败补偿不能释放全部已有 +资源，sandbox 会保留为可查询的 `RecoveryRequired`，后续可以再次执行销毁。 + +daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失败不会阻止 +其他记录继续处理，也不会阻止 API 启动。 + +正常关闭时，daemon 会先停止接收新请求并等待已有连接结束，再为每条持久化 +记录和仍持有的后端资源执行有界清理。单条清理失败不会跳过其余 sandbox， +所有未完成记录都会汇总报告。 + +操作记录只保存操作类型和开始时间，不记录每个资源步骤是否已经完成。中断的 +创建会被清理而不是从原位置继续，重启后也不会接管先前的后端进程。恢复失败 +后目前没有后台循环自动重试。checkpoint 和 reset 会先确认 sandbox 处于 +`Running` 且没有进行中的生命周期操作，然后返回 `501`；它们不会修改运行 +资源或持久化状态，其后端和存储操作尚未在这里实现。 + #### 健康检查 `GET /v1/health` 返回 daemon 状态，包含存储池就绪信息： From 2a0a6a3e877472cc560ce0c81a02b72586dec61c Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 20:34:04 +0800 Subject: [PATCH 19/40] feat(blaze): define sandbox guest protocol Managed sandboxes had no shared request and response format for communicating with a compatible guest agent. Define ping, exec, read, and write operations with request IDs, command inputs, file data, exit status, and standard-base64 outputs. Response IDs and outcome flags are required so a missing field cannot be mistaken for a confirmed rejection. This gives later transport and API commits one bounded wire contract to validate. It does not open a guest connection or expose a daemon route. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- .../crates/blaze-core/src/guest_protocol.rs | 132 ++++++++++++++++++ src/blaze/crates/blaze-core/src/lib.rs | 2 + 2 files changed, 134 insertions(+) create mode 100644 src/blaze/crates/blaze-core/src/guest_protocol.rs diff --git a/src/blaze/crates/blaze-core/src/guest_protocol.rs b/src/blaze/crates/blaze-core/src/guest_protocol.rs new file mode 100644 index 0000000000..282bfc2066 --- /dev/null +++ b/src/blaze/crates/blaze-core/src/guest_protocol.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Wire DTOs shared with a compatible sandbox guest agent. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +/// Firecracker vsock port used by the compatible sandbox guest agent. +pub const DEFAULT_GUEST_PORT: u32 = 5000; + +/// Maximum accepted JSON response line, excluding the newline delimiter. +pub const DEFAULT_MAX_RESPONSE_BYTES: usize = 32 * 1024 * 1024; + +/// Guest operation names implemented by `sandbox-agent`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum GuestOp { + /// Check whether the guest agent can serve requests. + Ping, + /// Execute one shell command. + Exec, + /// Read one guest file. + Read, + /// Replace one guest file. + Write, +} + +/// One newline-delimited request sent after the Firecracker CONNECT handshake. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GuestRequest { + /// Correlation identifier echoed by the guest. + pub id: String, + /// Requested guest operation. + pub op: GuestOp, + /// Shell command for [`GuestOp::Exec`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub cmd: Option, + /// Working directory for command execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub cwd: Option, + /// Environment additions for command execution. + #[serde(skip_serializing_if = "Option::is_none")] + pub env: Option>, + /// Guest-side timeout in seconds. + #[serde(skip_serializing_if = "Option::is_none")] + pub timeout: Option, + /// Guest path for file operations. + #[serde(skip_serializing_if = "Option::is_none")] + pub path: Option, + /// Standard-base64 file bytes for [`GuestOp::Write`]. + #[serde(skip_serializing_if = "Option::is_none")] + pub data_b64: Option, +} + +/// One newline-delimited response returned by the compatible guest agent. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GuestResponse { + /// Correlation identifier copied from the request. + pub id: String, + /// Whether the operation completed successfully. + pub ok: bool, + /// Guest error message when `ok` is false. + #[serde(default)] + pub err: Option, + /// Command exit status. + #[serde(default)] + pub rc: Option, + /// Standard-base64 command stdout. + #[serde(default)] + pub stdout_b64: Option, + /// Standard-base64 command stderr. + #[serde(default)] + pub stderr_b64: Option, + /// Standard-base64 file bytes. + #[serde(default)] + pub data_b64: Option, +} + +impl GuestRequest { + /// Build a request with operation-specific fields initially absent. + pub fn new(id: String, op: GuestOp) -> Self { + Self { + id, + op, + cmd: None, + cwd: None, + env: None, + timeout: None, + path: None, + data_b64: None, + } + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn request_uses_wire_operation_names_and_omits_absent_fields() { + let request = GuestRequest::new("request-1".to_string(), GuestOp::Exec); + + assert_eq!( + serde_json::to_value(request).expect("serialize request"), + json!({ + "id": "request-1", + "op": "exec", + }) + ); + } + + #[test] + fn response_requires_outcome_and_defaults_optional_fields() { + assert!(serde_json::from_value::(json!({"id": "request-1"})).is_err()); + assert!(serde_json::from_value::(json!({"ok": true})).is_err()); + let response: GuestResponse = serde_json::from_value(json!({ + "id": "request-1", + "ok": true + })) + .expect("deserialize response"); + + assert_eq!(response.id, "request-1"); + assert!(response.ok); + assert!(response.err.is_none()); + assert!(response.rc.is_none()); + assert!(response.stdout_b64.is_none()); + assert!(response.stderr_b64.is_none()); + assert!(response.data_b64.is_none()); + } +} diff --git a/src/blaze/crates/blaze-core/src/lib.rs b/src/blaze/crates/blaze-core/src/lib.rs index d95a9db18d..9ee302e5da 100644 --- a/src/blaze/crates/blaze-core/src/lib.rs +++ b/src/blaze/crates/blaze-core/src/lib.rs @@ -9,6 +9,7 @@ //! - [`config`]: daemon TOML configuration //! - [`policy`]: workload class + policy file schema //! - [`backend`]: backend kinds + selection / fallback +//! - [`guest_protocol`]: guest-agent wire DTOs //! - [`lifecycle`]: sandbox state machine + JSON persistence //! - [`pool`]: warm-pool key/stat/manager //! - [`template`]: template registry + refcnt + GC @@ -18,6 +19,7 @@ pub mod backend; pub mod config; pub mod error; +pub mod guest_protocol; pub mod kernel; pub mod lifecycle; pub mod policy; From 9dfe67fd200da37861aaccbdb42ea1a4952117c7 Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 22:20:36 +0800 Subject: [PATCH 20/40] feat(blaze): implement sandbox guest operations Managed sandboxes had no production path from the daemon API to a compatible guest endpoint, so callers could not execute commands or transfer files. Add a JSON-line client for the Firecracker vsock proxy with bounded inputs, deadlines, response frames, decoded outputs, and readiness polling. Exec and write failures become outcome-unknown only after delivery begins; pre-delivery failures and reads remain safe to retry. Let backend owners report an optional guest endpoint. SandboxManager holds the existing per-sandbox operation lock while it validates Running state and performs guest work. Cold create waits for readiness only when the selected owner reports an endpoint, and cancelled readiness runs the normal create compensation path. Expose exec, read, and write actions through both sandbox and compatible instance routes. Handlers enforce command, HTTP body, base64, decoded file, and response limits before delegating to the manager, and return stable error codes for known failures and unknown outcomes. This enables API callers to use bounded guest operations without bypassing runtime ownership. Guestless backends keep their create behavior and reject guest work. Warm activation does not repeat readiness, and the default 1 MiB HTTP limit must be raised to use the full 16 MiB decoded file limit. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/Cargo.lock | 14 + src/blaze/Cargo.toml | 3 +- src/blaze/crates/blazed/Cargo.toml | 1 + src/blaze/crates/blazed/src/api.rs | 680 ++++++++++- src/blaze/crates/blazed/src/daemon.rs | 1 + src/blaze/crates/blazed/src/error.rs | 42 +- src/blaze/crates/blazed/src/failpoint.rs | 10 + .../crates/blazed/src/failpoint_disabled.rs | 6 + src/blaze/crates/blazed/src/guest.rs | 60 + src/blaze/crates/blazed/src/guest/client.rs | 1051 +++++++++++++++++ src/blaze/crates/blazed/src/main.rs | 1 + .../crates/blazed/src/sandbox/manager.rs | 100 ++ src/blaze/crates/blazed/src/spawner.rs | 142 ++- .../crates/blazed/src/spawner/firecracker.rs | 56 +- 14 files changed, 2158 insertions(+), 9 deletions(-) create mode 100644 src/blaze/crates/blazed/src/guest.rs create mode 100644 src/blaze/crates/blazed/src/guest/client.rs diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 1ed8e6e6d1..297eb282a6 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -133,6 +133,7 @@ version = "0.3.0" dependencies = [ "anyhow", "async-trait", + "base64", "blaze-core", "chrono", "clap", @@ -309,6 +310,17 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.32" @@ -328,6 +340,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" dependencies = [ "futures-core", + "futures-macro", "futures-task", "pin-project-lite", "slab", @@ -987,6 +1000,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index 141359bb35..a67d5f6820 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -28,7 +28,7 @@ anyhow = "1.0" # Async runtime tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["net"] } +tokio-util = { version = "0.7", features = ["net", "rt"] } hyper = { version = "1", features = ["full"] } hyper-util = { version = "0.1", features = ["full"] } http-body-util = "0.1" @@ -42,6 +42,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } semver = "1.0" +base64 = "0.22" # Platform libc = "0.2" diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index 2082221b5d..67e1d58c4a 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -35,6 +35,7 @@ tracing-subscriber = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } libc = { workspace = true } +base64 = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 54753d1dfd..fe77d055c2 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -11,13 +11,15 @@ use std::str::FromStr; use std::sync::Arc; use std::time::Duration; +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; use blaze_core::backend::{BackendKind, BackendStatus, select_backend}; use blaze_core::kernel::HookKind; use blaze_core::lifecycle::{SandboxInstance, SandboxState, StartPath}; use blaze_core::policy::{ImageMetadata, RuntimeDecision, WorkloadClass, parse_duration}; use blaze_core::pool::{PoolConfig, PoolKey}; use http_body_util::Full; -use hyper::body::{Bytes, Incoming}; +use hyper::body::{Body, Bytes, Incoming}; use hyper::header::CONTENT_TYPE; use hyper::{Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; @@ -27,16 +29,30 @@ use tokio::time::Instant; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; +use crate::guest::MAX_GUEST_FILE_BYTES; use crate::request_body; use crate::sandbox::CreateSandbox; use crate::state::ServerState; +const MAX_EXEC_TIMEOUT_SECS: u32 = 20; + /// Top-level request handler. Always returns `Ok(Response)`; internal /// errors are turned into JSON error bodies so hyper never sees a panic. pub async fn handle( req: Request, state: Arc, ) -> std::result::Result>, Infallible> { + handle_request(req, state).await +} + +async fn handle_request( + req: Request, + state: Arc, +) -> std::result::Result>, Infallible> +where + B: Body + Unpin, + B::Error: std::fmt::Display, +{ state.metrics.inc(&state.metrics.requests_total); let method = req.method().clone(); @@ -63,6 +79,13 @@ pub async fn handle( Ok(resp) } +const fn max_base64_len(decoded_bytes: usize) -> usize { + decoded_bytes + .saturating_add(2) + .saturating_div(3) + .saturating_mul(4) +} + async fn dispatch( method: &Method, path: &str, @@ -86,6 +109,15 @@ async fn dispatch( ("GET", ["v1", "instances", id]) | ("GET", ["v1", "sandboxes", id]) => { get_instance(state, id) } + ("POST", ["v1", "sandboxes", id, "exec"]) | ("POST", ["v1", "instances", id, "exec"]) => { + exec_instance(state, id, &body).await + } + ("POST", ["v1", "sandboxes", id, "read"]) | ("POST", ["v1", "instances", id, "read"]) => { + read_instance_file(state, id, &body).await + } + ("POST", ["v1", "sandboxes", id, "write"]) | ("POST", ["v1", "instances", id, "write"]) => { + write_instance_file(state, id, &body).await + } ("POST", ["v1", "instances", id, "checkpoint"]) => checkpoint(state, id).await, ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, ("DELETE", ["v1", "instances", id]) @@ -331,6 +363,113 @@ async fn destroy_instance(state: &Arc, id: &str) -> Result, + #[serde(default)] + env: Option>, + #[serde(default)] + timeout: Option, +} + +async fn exec_instance( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: ExecRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid exec body: {error}")))?; + if request.cmd.is_empty() { + return Err(BlazeDaemonError::BadRequest( + "exec command is required".to_string(), + )); + } + let timeout = request.timeout.unwrap_or(MAX_EXEC_TIMEOUT_SECS); + if timeout == 0 || timeout > MAX_EXEC_TIMEOUT_SECS { + return Err(BlazeDaemonError::BadRequest(format!( + "exec timeout must be between 1 and {MAX_EXEC_TIMEOUT_SECS} seconds" + ))); + } + let result = state + .manager + .exec( + parse_uuid(id)?, + request.cmd, + request.cwd, + request.env, + timeout, + ) + .await?; + json_ok(&json!({ + "exit_code": result.exit_code, + "stdout_b64": BASE64.encode(result.stdout), + "stderr_b64": BASE64.encode(result.stderr), + })) +} + +#[derive(Debug, Deserialize)] +struct FileRequest { + path: String, + #[serde(default)] + data_b64: Option, +} + +async fn read_instance_file( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: FileRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid read body: {error}")))?; + let data = state + .manager + .read_file(parse_uuid(id)?, request.path) + .await?; + json_ok(&json!({"data_b64": BASE64.encode(data)})) +} + +async fn write_instance_file( + state: &Arc, + id: &str, + body: &[u8], +) -> Result>> { + let request: FileRequest = serde_json::from_slice(body) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid write body: {error}")))?; + let encoded = request + .data_b64 + .ok_or_else(|| BlazeDaemonError::BadRequest("data_b64 is required".to_string()))?; + let data = decode_guest_file(&encoded, MAX_GUEST_FILE_BYTES)?; + state + .manager + .write_file(parse_uuid(id)?, request.path, &data) + .await?; + json_ok(&json!({"written": true, "bytes": data.len()})) +} + +fn decode_guest_file(encoded: &str, limit: usize) -> Result> { + let encoded_limit = max_base64_len(limit); + if encoded.len() > encoded_limit { + return Err(crate::guest::GuestError::PayloadTooLarge { + actual: encoded.len(), + limit: encoded_limit, + } + .into()); + } + let data = BASE64 + .decode(encoded) + .map_err(|error| BlazeDaemonError::BadRequest(format!("invalid base64: {error}")))?; + if data.len() > limit { + return Err(crate::guest::GuestError::PayloadTooLarge { + actual: data.len(), + limit, + } + .into()); + } + Ok(data) +} + /// Stop every tracked sandbox after the daemon has stopped accepting work. /// /// Cleanup starts concurrently for all known owners and shares one deadline. @@ -670,10 +809,13 @@ fn json_response(status: StatusCode, value: &T) -> Result Response> { let status = StatusCode::from_u16(err.status_code()).unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - let body = json!({ + let mut body = json!({ "error": err.to_string(), "status": status.as_u16(), }); + if let Some(code) = err.api_code() { + body["code"] = json!(code); + } let bytes = serde_json::to_vec_pretty(&body) .unwrap_or_else(|_| br#"{"error":"serialize_failed"}"#.to_vec()); Response::builder() @@ -719,9 +861,11 @@ mod tests { use http_body_util::BodyExt; use crate::file_provider::FileStorageProvider; + #[cfg(target_os = "linux")] + use crate::spawner::BubblewrapSpawner; use crate::spawner::{ - BackendInstance, BackendSpawner, BubblewrapSpawner, DynBackendInstance, DynSpawner, - MockSpawner, SpawnFailure, SpawnResult, SpawnerRegistry, + BackendInstance, BackendSpawner, DynBackendInstance, DynSpawner, MockSpawner, SpawnFailure, + SpawnResult, SpawnerRegistry, }; use crate::state::ServerState; @@ -854,6 +998,32 @@ mod tests { (status, value) } + async fn handled_json( + state: &Arc, + method: Method, + path: &str, + body: Vec, + ) -> (StatusCode, serde_json::Value) { + let request = Request::builder() + .method(method) + .uri(path) + .header(hyper::header::CONTENT_LENGTH, body.len()) + .body(Full::new(Bytes::from(body))) + .expect("request"); + let response = handle_request(request, state.clone()) + .await + .expect("infallible response"); + let status = response.status(); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value = serde_json::from_slice(&body).expect("response json"); + (status, value) + } + struct TransientReconstructStorage { inner: FileStorageProvider, fail_reconstruct: AtomicBool, @@ -1248,6 +1418,72 @@ mod tests { } } + struct StalledGuestOwner { + instance_id: Uuid, + socket: PathBuf, + kill_count: Arc, + killed: AtomicBool, + } + + #[async_trait] + impl BackendInstance for StalledGuestOwner { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + fn guest_socket_path(&self) -> &Path { + &self.socket + } + + async fn try_wait(&self) -> blaze_core::Result> { + Ok(self.killed.load(Ordering::Acquire).then_some(SpawnResult { + instance_id: self.instance_id, + exit_code: Some(0), + signal: None, + })) + } + + async fn kill(&self) -> blaze_core::Result<()> { + if !self.killed.swap(true, Ordering::AcqRel) { + self.kill_count.fetch_add(1, Ordering::AcqRel); + } + Ok(()) + } + } + + struct StalledGuestSpawner { + spawned: Arc, + kill_count: Arc, + } + + #[async_trait] + impl BackendSpawner for StalledGuestSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + self.spawned.notify_one(); + Ok(Arc::new(StalledGuestOwner { + instance_id: request.instance_id, + socket: request.run_dir.join("missing-guest.uds"), + kill_count: self.kill_count.clone(), + killed: AtomicBool::new(false), + })) + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + _instance_id: Uuid, + _run_dir: &Path, + ) -> blaze_core::Result<()> { + Ok(()) + } + } + struct CountingStorage { inner: FileStorageProvider, release_count: Arc, @@ -1749,6 +1985,48 @@ mod tests { ); } + #[tokio::test] + async fn guest_manager_waits_for_the_lifecycle_lock() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + + let operation = state.manager.operation_lock(uuid).lock_owned().await; + let exec_state = state.clone(); + let mut pending_exec = tokio::spawn(async move { + exec_state + .manager + .exec(uuid, "printf guest-lock".into(), None, None, 5) + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut pending_exec) + .await + .is_err(), + "guest operation bypassed the lifecycle lock" + ); + drop(operation); + + let exec = pending_exec + .await + .expect("exec task") + .expect("managed exec"); + assert_eq!(exec.stdout, b"printf guest-lock"); + } + #[tokio::test] async fn manager_cleanup_releases_tracked_runtime_resources() { let temp = tempfile::tempdir().expect("temp"); @@ -1915,6 +2193,309 @@ mod tests { ); } + #[tokio::test] + async fn sandbox_guest_routes_use_owned_runtime() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + + let (status, exec) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/exec"), + serde_json::to_vec(&json!({ + "cmd": "printf routed", + "timeout": 5, + })) + .expect("exec request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(exec["exit_code"], 0); + assert_eq!(exec["stdout_b64"], BASE64.encode(b"printf routed")); + + let encoded = "AAEC/2d1ZXN0"; + let (status, written) = dispatched_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/write"), + serde_json::to_vec(&json!({ + "path": "/tmp/value", + "data_b64": encoded, + })) + .expect("write request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], 9); + + let (status, read) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/read"), + serde_json::to_vec(&json!({"path": "/tmp/value"})).expect("read request"), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(read["data_b64"], encoded); + + let invalid_timeout = dispatch( + &Method::POST, + &format!("/v1/sandboxes/{id}/exec"), + "", + serde_json::to_vec(&json!({ + "cmd": "true", + "timeout": MAX_EXEC_TIMEOUT_SECS + 1, + })) + .expect("invalid request"), + &state, + ) + .await + .expect_err("timeout above the API limit must fail"); + assert!(matches!(invalid_timeout, BlazeDaemonError::BadRequest(_))); + + assert_eq!( + decode_guest_file(&BASE64.encode(b"1234"), 4).expect("boundary"), + b"1234" + ); + assert!(matches!( + decode_guest_file(&BASE64.encode(b"12345"), 4), + Err(BlazeDaemonError::Guest( + crate::guest::GuestError::PayloadTooLarge { .. } + )) + )); + assert!(matches!( + decode_guest_file("not/base64!", 16), + Err(BlazeDaemonError::BadRequest(_)) + )); + + let (status, destroyed) = dispatched_json( + &state, + Method::DELETE, + &format!("/v1/sandboxes/{id}"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(destroyed["destroyed"], true); + } + + #[tokio::test] + async fn guest_write_respects_http_and_decoded_limits() { + const EXTENDED_BODY_LIMIT: usize = 22 * 1024 * 1024; + + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let default_limit = config.api.max_body_bytes; + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let path = format!("/v1/sandboxes/{id}/write"); + + let envelope_payload = vec![b'y'; 800 * 1024]; + let envelope_body = serde_json::to_vec(&json!({ + "path": "/tmp/http-envelope", + "data_b64": BASE64.encode(&envelope_payload), + })) + .expect("write request above the default HTTP limit"); + assert!(envelope_body.len() > default_limit); + let (status, error) = + handled_json(&state, Method::POST, &path, envelope_body.clone()).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error["status"], 413); + + state.config.lock().expect("config").api.max_body_bytes = EXTENDED_BODY_LIMIT; + let (status, written) = handled_json(&state, Method::POST, &path, envelope_body).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], envelope_payload.len()); + assert_eq!( + state + .manager + .read_file(uuid, "/tmp/http-envelope".into()) + .await + .expect("read envelope payload"), + envelope_payload + ); + + let mut payload = vec![b'z'; MAX_GUEST_FILE_BYTES]; + let body = serde_json::to_vec(&json!({ + "path": "/tmp/max-size", + "data_b64": BASE64.encode(&payload), + })) + .expect("write request"); + assert!(body.len() <= EXTENDED_BODY_LIMIT); + + let (status, written) = handled_json(&state, Method::POST, &path, body).await; + assert_eq!(status, StatusCode::OK); + assert_eq!(written["bytes"], MAX_GUEST_FILE_BYTES); + let readback = state + .manager + .read_file(uuid, "/tmp/max-size".into()) + .await + .expect("read maximum file"); + assert_eq!(readback, payload); + drop(readback); + + payload.push(b'z'); + let oversized = serde_json::to_vec(&json!({ + "path": "/tmp/too-large", + "data_b64": BASE64.encode(&payload), + })) + .expect("oversized write request"); + assert!(oversized.len() <= EXTENDED_BODY_LIMIT); + let (status, error) = handled_json(&state, Method::POST, &path, oversized).await; + assert_eq!(status, StatusCode::PAYLOAD_TOO_LARGE); + assert_eq!(error["status"], 413); + } + + #[tokio::test] + async fn write_route_reports_unknown_after_delivery_failure() { + use tokio::io::{AsyncBufReadExt, AsyncWriteExt}; + + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("instance id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .backend_owner(uuid) + .expect("mock owner") + .kill() + .await + .expect("stop mock guest"); + + let socket = temp.path().join("uncertain.uds"); + let listener = tokio::net::UnixListener::bind(&socket).expect("bind guest endpoint"); + state + .manager + .insert_backend_owner( + uuid, + Arc::new(StalledGuestOwner { + instance_id: uuid, + socket, + kill_count: Arc::new(AtomicUsize::new(0)), + killed: AtomicBool::new(false), + }), + ) + .expect("replace backend owner"); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept guest request"); + let mut reader = tokio::io::BufReader::new(stream); + let mut connect = String::new(); + reader.read_line(&mut connect).await.expect("read connect"); + assert_eq!(connect, "CONNECT 5000\n"); + reader + .get_mut() + .write_all(b"OK 5000\n") + .await + .expect("write handshake"); + let mut request = String::new(); + reader + .read_line(&mut request) + .await + .expect("read guest request"); + let request: serde_json::Value = + serde_json::from_str(&request).expect("parse guest request"); + assert_eq!(request["op"], "write"); + }); + + let body = serde_json::to_vec(&json!({ + "path": "/tmp/value", + "data_b64": BASE64.encode(b"value"), + })) + .expect("write request"); + let (status, error) = handled_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/write"), + body, + ) + .await; + assert_eq!(status, StatusCode::GATEWAY_TIMEOUT); + assert_eq!(error["code"], "guest_outcome_unknown"); + server.await.expect("guest server"); + } + + #[tokio::test] + async fn unknown_guest_outcome_has_stable_api_code() { + let response = error_response(&BlazeDaemonError::Guest( + crate::guest::GuestError::OutcomeUnknown("response lost".into()), + )); + assert_eq!(response.status(), StatusCode::GATEWAY_TIMEOUT); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_outcome_unknown"); + assert_eq!(value["status"], 504); + + let response = error_response(&BlazeDaemonError::Guest( + crate::guest::GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + )); + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_response_too_large"); + + let response = error_response(&BlazeDaemonError::Guest(crate::guest::GuestError::Timeout( + "connect stalled".into(), + ))); + let body = response + .into_body() + .collect() + .await + .expect("response body") + .to_bytes(); + let value: serde_json::Value = serde_json::from_slice(&body).expect("error json"); + assert_eq!(value["code"], "guest_timeout"); + } + #[tokio::test] async fn create_publishes_ownership_before_provider_acquire() { let temp = tempfile::tempdir().expect("temp"); @@ -1972,6 +2553,66 @@ mod tests { assert!(!instances_dir.join(id.to_string()).exists()); } + #[tokio::test] + async fn shutdown_cancels_readiness_and_completes_create_compensation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + let spawned = Arc::new(tokio::sync::Notify::new()); + let kill_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(StalledGuestSpawner { + spawned: spawned.clone(), + kill_count: kill_count.clone(), + }), + ), + BackendKind::Mock, + storage, + ); + let create_state = state.clone(); + let create = + tokio::spawn(async move { create_instance(&create_state, &test_request()).await }); + tokio::time::timeout(Duration::from_secs(1), spawned.notified()) + .await + .expect("backend started"); + + state.manager.begin_shutdown(); + let error = tokio::time::timeout(Duration::from_secs(1), create) + .await + .expect("create cancellation") + .expect("create task") + .expect_err("readiness must be cancelled"); + assert!(matches!( + error, + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) + )); + + let instance = state + .manager + .list() + .expect("instances") + .into_iter() + .next() + .expect("cancelled create record"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert!(instance.operation.is_none()); + assert_eq!(instance.backend_ownership, BackendOwnership::Stopped); + assert!(state.manager.backend_owner(instance.id).is_none()); + assert_eq!(kill_count.load(Ordering::Acquire), 1); + assert!(!instances_dir.join(instance.id.to_string()).exists()); + shutdown_instances(&state, Duration::from_millis(100)) + .await + .expect("no ownership remains"); + } + #[tokio::test(start_paused = true)] async fn daemon_shutdown_joins_all_owners_within_one_budget() { let temp = tempfile::tempdir().expect("temp"); @@ -2440,6 +3081,37 @@ mod tests { assert_warm_state_commit_failure_restores_claim("warm-final-state-commit").await; } + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn guest_readiness_failure_compensates_owned_resources() { + let request = test_request(); + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let hook = crate::failpoint::TestFailpoint::new(&["create-guest-ready"]); + + hook.run(create_instance(&state, &request)) + .await + .expect_err("guest readiness failure"); + + let instance = state + .instances + .lock() + .expect("instances") + .values() + .next() + .cloned() + .expect("destroyed create"); + assert_eq!(instance.state, SandboxState::Destroyed); + assert!(state.manager.backend_owner(instance.id).is_none()); + assert!( + !temp + .path() + .join("instances") + .join(instance.id.to_string()) + .exists() + ); + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn failure_hooks_drive_create_and_destroy_compensation() { diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 2a30ddf85f..f0050fa789 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -397,6 +397,7 @@ async fn serve( uds.stop_accepting(); drop(tcp); + state.manager.begin_shutdown(); let drain = connections.shutdown(SHUTDOWN_BUDGET.connection_drain); let cleanup = api::shutdown_instances(&state, SHUTDOWN_BUDGET.runtime_cleanup); let shutdown_result = finish_shutdown(drain, cleanup).await; diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index 560783115c..4afd788c22 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//! Local errors for the blazed binary (daemon + CLI client). +//! Local errors for the daemon binary and HTTP API. //! //! Wraps [`blaze_core::BlazeError`] so the daemon can additionally //! surface I/O, hyper, and CLI-side failures without expanding the @@ -16,6 +16,9 @@ pub enum BlazeDaemonError { #[error("core error: {0}")] Core(#[from] blaze_core::BlazeError), + #[error(transparent)] + Guest(#[from] crate::guest::GuestError), + #[error("io error: {0}")] Io(#[from] std::io::Error), @@ -94,6 +97,37 @@ pub enum BlazeDaemonError { } impl BlazeDaemonError { + /// Stable machine-readable code for errors that callers must branch on. + pub fn api_code(&self) -> Option<&'static str> { + match self { + BlazeDaemonError::Guest(crate::guest::GuestError::Io(_)) => { + Some("guest_transport_error") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Json(_)) + | BlazeDaemonError::Guest(crate::guest::GuestError::Protocol(_)) => { + Some("guest_response_invalid") + } + BlazeDaemonError::Guest(crate::guest::GuestError::InvalidArgument(_)) => { + Some("guest_invalid_request") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Timeout(_)) => Some("guest_timeout"), + BlazeDaemonError::Guest(crate::guest::GuestError::OutcomeUnknown(_)) => { + Some("guest_outcome_unknown") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Rejected(_)) => { + Some("guest_rejected") + } + BlazeDaemonError::Guest(crate::guest::GuestError::PayloadTooLarge { .. }) => { + Some("guest_request_too_large") + } + BlazeDaemonError::Guest(crate::guest::GuestError::ResponseTooLarge { .. }) => { + Some("guest_response_too_large") + } + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) => Some("guest_cancelled"), + _ => None, + } + } + /// HTTP status code that should accompany this error in API responses. pub fn status_code(&self) -> u16 { match self { @@ -108,6 +142,12 @@ impl BlazeDaemonError { | BlazeDaemonError::Core(blaze_core::BlazeError::InvalidStateTransition { .. }) => 422, BlazeDaemonError::Core(blaze_core::BlazeError::OperationInProgress { .. }) => 409, BlazeDaemonError::Core(blaze_core::BlazeError::BackendUnavailable { .. }) => 503, + BlazeDaemonError::Guest(crate::guest::GuestError::InvalidArgument(_)) => 400, + BlazeDaemonError::Guest(crate::guest::GuestError::Timeout(_)) => 504, + BlazeDaemonError::Guest(crate::guest::GuestError::OutcomeUnknown(_)) => 504, + BlazeDaemonError::Guest(crate::guest::GuestError::PayloadTooLarge { .. }) => 413, + BlazeDaemonError::Guest(crate::guest::GuestError::Cancelled) => 503, + BlazeDaemonError::Guest(_) => 502, _ => 500, } } diff --git a/src/blaze/crates/blazed/src/failpoint.rs b/src/blaze/crates/blazed/src/failpoint.rs index 8338d78660..15edb0bc92 100644 --- a/src/blaze/crates/blazed/src/failpoint.rs +++ b/src/blaze/crates/blazed/src/failpoint.rs @@ -122,6 +122,16 @@ pub(crate) fn storage(name: &str) -> blaze_core::Result<()> { Ok(()) } +/// Return a guest-domain error when `name` is currently armed. +pub(crate) fn guest(name: &str) -> crate::guest::Result<()> { + if hit(name) { + return Err(crate::guest::GuestError::Rejected(format!( + "test failpoint '{name}' triggered" + ))); + } + Ok(()) +} + /// Return a daemon state-commit error when `name` is currently armed. pub(crate) fn state(name: &str) -> crate::error::Result<()> { if hit(name) { diff --git a/src/blaze/crates/blazed/src/failpoint_disabled.rs b/src/blaze/crates/blazed/src/failpoint_disabled.rs index 049aa05ab9..a43b1ef25f 100644 --- a/src/blaze/crates/blazed/src/failpoint_disabled.rs +++ b/src/blaze/crates/blazed/src/failpoint_disabled.rs @@ -16,6 +16,11 @@ pub(crate) fn storage(_name: &str) -> blaze_core::Result<()> { Ok(()) } +/// Leave guest operations unchanged in production builds. +pub(crate) fn guest(_name: &str) -> crate::guest::Result<()> { + Ok(()) +} + /// Leave state commits unchanged in production builds. pub(crate) fn state(_name: &str) -> crate::error::Result<()> { Ok(()) @@ -31,6 +36,7 @@ mod tests { super::announce(); super::backend("any").expect("backend hook"); super::storage("any").expect("storage hook"); + super::guest("any").expect("guest hook"); super::state("any").expect("state hook"); super::pause("any").await; } diff --git a/src/blaze/crates/blazed/src/guest.rs b/src/blaze/crates/blazed/src/guest.rs new file mode 100644 index 0000000000..2b92cc1b89 --- /dev/null +++ b/src/blaze/crates/blazed/src/guest.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Firecracker-vsock guest agent client. + +pub mod client; + +pub use client::GuestClient; +pub use client::GuestExecResult; + +use thiserror::Error; + +/// Maximum decoded file payload accepted by guest read and write operations. +pub(crate) const MAX_GUEST_FILE_BYTES: usize = 16 * 1024 * 1024; + +/// Guest protocol and transport failures. +#[derive(Debug, Error)] +pub enum GuestError { + /// Unix socket I/O failed. + #[error("guest transport error: {0}")] + Io(#[from] std::io::Error), + /// JSON encoding or decoding failed. + #[error("guest JSON error: {0}")] + Json(#[from] serde_json::Error), + /// Firecracker vsock or guest framing was invalid. + #[error("guest protocol error: {0}")] + Protocol(String), + /// Caller supplied an invalid guest operation argument. + #[error("invalid guest request: {0}")] + InvalidArgument(String), + /// A bounded guest operation timed out. + #[error("guest operation timed out: {0}")] + Timeout(String), + /// A state-changing request timed out after it may have reached the guest. + #[error("guest operation outcome is unknown: {0}")] + OutcomeUnknown(String), + /// The guest returned an application error. + #[error("guest operation failed: {0}")] + Rejected(String), + /// Caller-supplied decoded file data exceeded the guest file hard limit. + #[error("guest payload too large: {actual} bytes exceeds {limit}")] + PayloadTooLarge { + /// Decoded or framed byte count. + actual: usize, + /// Configured limit. + limit: usize, + }, + /// A guest response exceeded the bounded frame or decoded output limit. + #[error("guest response too large: {actual} bytes exceeds {limit}")] + ResponseTooLarge { + /// Decoded or framed byte count. + actual: usize, + /// Configured limit. + limit: usize, + }, + /// Readiness polling was cancelled during daemon shutdown. + #[error("guest readiness wait cancelled")] + Cancelled, +} + +/// Result alias for guest operations. +pub type Result = std::result::Result; diff --git a/src/blaze/crates/blazed/src/guest/client.rs b/src/blaze/crates/blazed/src/guest/client.rs new file mode 100644 index 0000000000..13dd2e91db --- /dev/null +++ b/src/blaze/crates/blazed/src/guest/client.rs @@ -0,0 +1,1051 @@ +// SPDX-License-Identifier: Apache-2.0 +//! JSON-line client layered over Firecracker's vsock Unix socket proxy. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +use base64::Engine; +use base64::engine::general_purpose::STANDARD as BASE64; +use blaze_core::guest_protocol::{ + DEFAULT_GUEST_PORT, DEFAULT_MAX_RESPONSE_BYTES, GuestOp, GuestRequest, GuestResponse, +}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixStream; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use super::{GuestError, Result}; + +const READY_ATTEMPT_TIMEOUT: Duration = Duration::from_millis(250); +const PROTOCOL_GRACE: Duration = Duration::from_secs(10); +const MAX_EXEC_COMMAND_BYTES: usize = 64 * 1024; +const MAX_EXEC_CWD_BYTES: usize = 4096; +const MAX_EXEC_ENV_ENTRIES: usize = 256; +const MAX_EXEC_ENV_BYTES: usize = 64 * 1024; + +/// Result of one command executed by the guest agent. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GuestExecResult { + /// Guest process exit status. + pub exit_code: i32, + /// Decoded stdout bytes. + pub stdout: Vec, + /// Decoded stderr bytes. + pub stderr: Vec, +} + +/// Client for one Firecracker guest agent. +#[derive(Debug, Clone)] +pub struct GuestClient { + vsock_path: PathBuf, + port: u32, + io_timeout: Duration, + max_response_bytes: usize, + max_file_bytes: usize, +} + +impl GuestClient { + /// Create a client with production protocol defaults. + pub fn new(vsock_path: PathBuf, io_timeout: Duration, max_file_bytes: usize) -> Self { + Self { + vsock_path, + port: DEFAULT_GUEST_PORT, + io_timeout, + max_response_bytes: DEFAULT_MAX_RESPONSE_BYTES, + max_file_bytes, + } + } + + /// Override protocol limits for focused tests. + #[cfg(test)] + fn with_response_limit(mut self, max_response_bytes: usize) -> Self { + self.max_response_bytes = max_response_bytes; + self + } + + /// Check whether the guest agent is responsive. + pub async fn ping(&self) -> Result<()> { + let request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Ping); + self.send_recv(&request).await?; + Ok(()) + } + + /// Poll readiness with bounded exponential backoff. + pub async fn wait_ready( + &self, + deadline: Duration, + cancellation: &CancellationToken, + ) -> Result<()> { + let started = Instant::now(); + let mut backoff = Duration::from_millis(10); + let mut last_error = None; + while started.elapsed() < deadline { + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + break; + } + let attempt_timeout = READY_ATTEMPT_TIMEOUT.min(remaining); + tokio::select! { + _ = cancellation.cancelled() => return Err(GuestError::Cancelled), + result = tokio::time::timeout(attempt_timeout, self.ping()) => { + match result { + Ok(Ok(())) => return Ok(()), + Ok(Err(error)) => last_error = Some(error), + Err(_) => { + last_error = Some(GuestError::Timeout(format!( + "readiness ping exceeded {attempt_timeout:?}" + ))); + } + } + } + } + let remaining = deadline.saturating_sub(started.elapsed()); + if remaining.is_zero() { + break; + } + tokio::select! { + _ = cancellation.cancelled() => return Err(GuestError::Cancelled), + _ = tokio::time::sleep(backoff.min(remaining)) => {} + } + backoff = (backoff * 2).min(Duration::from_millis(250)); + } + Err(GuestError::Timeout(format!( + "guest at {} was not ready within {:?}: {}", + self.vsock_path.display(), + deadline, + last_error + .map(|error| error.to_string()) + .unwrap_or_else(|| "no attempt completed".to_string()) + ))) + } + + /// Execute one shell command in the guest. + pub async fn exec( + &self, + command: String, + cwd: Option, + env: Option>, + timeout_secs: u32, + ) -> Result { + validate_exec_inputs(&command, cwd.as_deref(), env.as_ref())?; + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Exec); + request.cmd = Some(command); + request.cwd = Some(cwd.unwrap_or_else(|| "/".to_string())); + request.env = env; + request.timeout = Some(timeout_secs); + let timeout = operation_timeout(timeout_secs); + if timeout > self.io_timeout { + return Err(GuestError::InvalidArgument(format!( + "exec timeout plus protocol grace ({timeout:?}) exceeds configured limit {:?}", + self.io_timeout + ))); + } + let response = self.send_recv_with_timeout(&request, timeout).await?; + let exit_code = response + .rc + .ok_or_else(|| { + GuestError::Protocol("successful exec response is missing rc".to_string()) + }) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + let stdout = decode_limited( + response.stdout_b64.as_deref().unwrap_or_default(), + self.max_response_bytes, + ) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + let stderr = decode_limited( + response.stderr_b64.as_deref().unwrap_or_default(), + self.max_response_bytes, + ) + .map_err(|error| classify_after_request(GuestOp::Exec, error))?; + Ok(GuestExecResult { + exit_code, + stdout, + stderr, + }) + } + + /// Read one guest file. + pub async fn read_file(&self, path: String) -> Result> { + validate_guest_path(&path)?; + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Read); + request.path = Some(path); + let guest_timeout = request_timeout_secs(self.io_timeout); + request.timeout = Some(guest_timeout); + let response = self + .send_recv_with_timeout( + &request, + operation_timeout(guest_timeout).min(self.io_timeout), + ) + .await?; + let data = response.data_b64.as_deref().ok_or_else(|| { + GuestError::Protocol("successful read response is missing data_b64".to_string()) + })?; + decode_limited(data, self.max_file_bytes) + } + + /// Replace one guest file. + pub async fn write_file(&self, path: String, data: &[u8]) -> Result<()> { + validate_guest_path(&path)?; + if data.len() > self.max_file_bytes { + return Err(GuestError::PayloadTooLarge { + actual: data.len(), + limit: self.max_file_bytes, + }); + } + let mut request = GuestRequest::new(Uuid::new_v4().to_string(), GuestOp::Write); + request.path = Some(path); + request.data_b64 = Some(BASE64.encode(data)); + let guest_timeout = request_timeout_secs(self.io_timeout); + request.timeout = Some(guest_timeout); + self.send_recv_with_timeout( + &request, + operation_timeout(guest_timeout).min(self.io_timeout), + ) + .await?; + Ok(()) + } + + async fn send_recv(&self, request: &GuestRequest) -> Result { + self.send_recv_with_timeout(request, self.io_timeout).await + } + + async fn send_recv_with_timeout( + &self, + request: &GuestRequest, + timeout: Duration, + ) -> Result { + let mut encoded = serde_json::to_vec(request)?; + encoded.push(b'\n'); + + let started = Instant::now(); + let mut stream = match tokio::time::timeout(timeout, self.connect_guest()).await { + Ok(result) => result?, + Err(_) => return Err(self.timeout_error(request, timeout, "before request delivery")), + }; + let remaining = timeout.saturating_sub(started.elapsed()); + if remaining.is_zero() { + return Err(self.timeout_error(request, timeout, "before request delivery")); + } + + match tokio::time::timeout( + remaining, + self.exchange_request(&mut stream, request, &encoded), + ) + .await + { + Ok(Ok(response)) => Ok(response), + Ok(Err(error)) => Err(classify_after_request(request.op, error)), + Err(_) => Err(classify_after_request( + request.op, + self.timeout_error(request, timeout, "after request delivery began"), + )), + } + } + + async fn connect_guest(&self) -> Result { + let mut stream = UnixStream::connect(&self.vsock_path).await?; + stream + .write_all(format!("CONNECT {}\n", self.port).as_bytes()) + .await?; + let handshake = read_line(&mut stream, 128).await?; + let handshake = std::str::from_utf8(&handshake).map_err(|error| { + GuestError::Protocol(format!("CONNECT response is not UTF-8: {error}")) + })?; + let peer_cid = handshake + .strip_prefix("OK ") + .and_then(|value| value.parse::().ok()); + if peer_cid.is_none() { + return Err(GuestError::Protocol(format!( + "unexpected CONNECT {} response: expected \"OK \", received {handshake:?}", + self.port, + ))); + } + Ok(stream) + } + + async fn exchange_request( + &self, + stream: &mut UnixStream, + request: &GuestRequest, + encoded: &[u8], + ) -> Result { + stream.write_all(encoded).await?; + stream.flush().await?; + let line = read_line(stream, self.max_response_bytes).await?; + let response: GuestResponse = serde_json::from_slice(&line)?; + if response.id != request.id { + return Err(GuestError::Protocol(format!( + "response id mismatch: sent {}, received {}", + request.id, response.id + ))); + } + if !response.ok { + return Err(GuestError::Rejected(response.err.unwrap_or_else(|| { + "guest rejected request without an error".to_string() + }))); + } + Ok(response) + } + + fn timeout_error(&self, request: &GuestRequest, timeout: Duration, phase: &str) -> GuestError { + GuestError::Timeout(format!( + "{:?} request to {} exceeded {:?} {phase}", + request.op, + self.vsock_path.display(), + timeout + )) + } +} + +fn classify_after_request(operation: GuestOp, error: GuestError) -> GuestError { + if matches!(operation, GuestOp::Exec | GuestOp::Write) + && !matches!( + error, + GuestError::Rejected(_) | GuestError::OutcomeUnknown(_) + ) + { + GuestError::OutcomeUnknown(error.to_string()) + } else { + error + } +} + +fn operation_timeout(guest_timeout_secs: u32) -> Duration { + Duration::from_secs(u64::from(guest_timeout_secs)).saturating_add(PROTOCOL_GRACE) +} + +fn request_timeout_secs(io_timeout: Duration) -> u32 { + io_timeout + .saturating_sub(PROTOCOL_GRACE) + .as_secs() + .clamp(1, u64::from(u32::MAX)) as u32 +} + +async fn read_line(stream: &mut R, limit: usize) -> Result> +where + R: AsyncRead + Unpin, +{ + let bounded = limit.saturating_add(1); + let mut reader = BufReader::new(stream).take(bounded as u64); + let mut output = Vec::with_capacity(limit.min(8192)); + let count = reader.read_until(b'\n', &mut output).await?; + if output.last() == Some(&b'\n') { + output.pop(); + if output.len() <= limit { + return Ok(output); + } + } + if output.len() > limit { + return Err(GuestError::ResponseTooLarge { + actual: output.len(), + limit, + }); + } + debug_assert_eq!(count, output.len()); + Err(GuestError::Protocol( + "connection closed before newline delimiter".to_string(), + )) +} + +fn decode_limited(encoded: &str, limit: usize) -> Result> { + let encoded_limit = limit.div_ceil(3).saturating_mul(4); + if encoded.len() > encoded_limit { + return Err(GuestError::ResponseTooLarge { + actual: encoded.len(), + limit: encoded_limit, + }); + } + let decoded = BASE64 + .decode(encoded) + .map_err(|error| GuestError::Protocol(format!("invalid base64 payload: {error}")))?; + if decoded.len() > limit { + return Err(GuestError::ResponseTooLarge { + actual: decoded.len(), + limit, + }); + } + Ok(decoded) +} + +fn validate_exec_inputs( + command: &str, + cwd: Option<&str>, + env: Option<&HashMap>, +) -> Result<()> { + if command.is_empty() { + return Err(GuestError::InvalidArgument( + "exec command is empty".to_string(), + )); + } + if command.len() > MAX_EXEC_COMMAND_BYTES || command.contains('\0') { + return Err(GuestError::InvalidArgument(format!( + "exec command must be NUL-free and at most {MAX_EXEC_COMMAND_BYTES} bytes" + ))); + } + if let Some(cwd) = cwd + && (cwd.len() > MAX_EXEC_CWD_BYTES || cwd.contains('\0')) + { + return Err(GuestError::InvalidArgument(format!( + "exec cwd must be NUL-free and at most {MAX_EXEC_CWD_BYTES} bytes" + ))); + } + if let Some(env) = env { + if env.len() > MAX_EXEC_ENV_ENTRIES { + return Err(GuestError::InvalidArgument(format!( + "exec environment has {} entries; limit is {MAX_EXEC_ENV_ENTRIES}", + env.len() + ))); + } + let mut total = 0_usize; + for (key, value) in env { + if key.is_empty() || key.contains('=') || key.contains('\0') || value.contains('\0') { + return Err(GuestError::InvalidArgument( + "exec environment keys must be non-empty and contain no '=', and keys and \ + values must be NUL-free" + .to_string(), + )); + } + total = total + .checked_add(key.len()) + .and_then(|bytes| bytes.checked_add(value.len())) + .ok_or_else(|| { + GuestError::InvalidArgument("exec environment size overflow".to_string()) + })?; + if total > MAX_EXEC_ENV_BYTES { + return Err(GuestError::InvalidArgument(format!( + "exec environment is {total} bytes; limit is {MAX_EXEC_ENV_BYTES}" + ))); + } + } + } + Ok(()) +} + +fn validate_guest_path(path: &str) -> Result<()> { + if path.is_empty() || !path.starts_with('/') || path.len() > 4096 || path.contains('\0') { + return Err(GuestError::InvalidArgument( + "guest file path must be absolute, NUL-free, and at most 4096 bytes".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use serde_json::json; + use tokio::net::UnixListener; + + use super::*; + + async fn spawn_server( + socket: PathBuf, + response: Arc serde_json::Value + Send + Sync>, + ) { + let listener = UnixListener::bind(socket).expect("bind"); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let response = response.clone(); + tokio::spawn(async move { + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 1073742006\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request: serde_json::Value = + serde_json::from_slice(&request).expect("json"); + let response = response(request); + let mut bytes = serde_json::to_vec(&response).expect("encode"); + bytes.push(b'\n'); + stream.write_all(&bytes).await.expect("write"); + }); + } + }); + } + + async fn accept_request(listener: &UnixListener) -> (UnixStream, serde_json::Value) { + let (mut stream, _) = listener.accept().await.expect("accept"); + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 1073742006\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request = serde_json::from_slice(&request).expect("json"); + (stream, request) + } + + #[test] + fn side_effect_errors_become_unknown_only_after_delivery_starts() { + for operation in [GuestOp::Exec, GuestOp::Write] { + assert!(matches!( + classify_after_request(operation, GuestError::Protocol("EOF".into())), + GuestError::OutcomeUnknown(_) + )); + assert!(matches!( + classify_after_request( + operation, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + ), + GuestError::OutcomeUnknown(_) + )); + assert!(matches!( + classify_after_request(operation, GuestError::Rejected("denied".into())), + GuestError::Rejected(_) + )); + } + + assert!(matches!( + classify_after_request(GuestOp::Read, GuestError::Protocol("EOF".into())), + GuestError::Protocol(_) + )); + assert!(matches!( + classify_after_request( + GuestOp::Read, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4, + }, + ), + GuestError::ResponseTooLarge { .. } + )); + } + + #[tokio::test] + async fn ping_exec_read_and_write_follow_existing_protocol() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let requests = Arc::new(AtomicUsize::new(0)); + let server_requests = requests.clone(); + spawn_server( + socket.clone(), + Arc::new(move |request| { + server_requests.fetch_add(1, Ordering::Relaxed); + match request["op"].as_str().expect("op") { + "exec" => json!({ + "id": request["id"], + "ok": true, + "rc": 7, + "stdout_b64": BASE64.encode(b"out"), + "stderr_b64": BASE64.encode(b"err") + }), + "read" => { + assert_eq!(request["timeout"], 5); + json!({ + "id": request["id"], + "ok": true, + "data_b64": BASE64.encode(b"data") + }) + } + "write" => { + assert_eq!(request["timeout"], 5); + json!({"id": request["id"], "ok": true}) + } + _ => json!({"id": request["id"], "ok": true}), + } + }), + ) + .await; + let client = GuestClient::new(socket, Duration::from_secs(15), 1024); + client.ping().await.expect("ping"); + let exec = client + .exec("exit 7".into(), None, None, 1) + .await + .expect("exec"); + assert_eq!(exec.exit_code, 7); + assert_eq!(exec.stdout, b"out"); + assert_eq!( + client.read_file("/tmp/x".into()).await.expect("read"), + b"data" + ); + client + .write_file("/tmp/x".into(), b"replacement") + .await + .expect("write"); + assert_eq!(requests.load(Ordering::Relaxed), 4); + } + + #[tokio::test] + async fn mismatched_response_id_is_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|_| json!({"id": "wrong", "ok": true})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("mismatch"); + assert!(matches!(error, GuestError::Protocol(_))); + } + + #[tokio::test] + async fn malformed_json_is_rejected_without_poisoning_the_next_call() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (mut first, _) = accept_request(&listener).await; + first.write_all(b"{not-json}\n").await.expect("malformed"); + + let (mut second, request) = accept_request(&listener).await; + let mut response = + serde_json::to_vec(&json!({"id": request["id"], "ok": true})).expect("response"); + response.push(b'\n'); + second.write_all(&response).await.expect("valid"); + }); + let client = GuestClient::new(socket, Duration::from_secs(1), 1024); + assert!(matches!(client.ping().await, Err(GuestError::Json(_)))); + client.ping().await.expect("subsequent request"); + server.await.expect("server task"); + } + + #[tokio::test] + async fn missing_socket_is_reported_as_connection_failure() { + let temp = tempfile::tempdir().expect("temp"); + let client = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_millis(100), + 1024, + ); + let error = client.ping().await.expect_err("connection failure"); + assert!(matches!(error, GuestError::Io(_))); + + let write_error = client + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("write did not reach the guest"); + assert!(matches!(write_error, GuestError::Io(_))); + } + + #[tokio::test] + async fn invalid_exec_timeout_is_a_caller_error() { + let temp = tempfile::tempdir().expect("temp"); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(15), + 1024, + ) + .exec("true".into(), None, None, 6) + .await + .expect_err("timeout exceeds request budget"); + assert!(matches!(error, GuestError::InvalidArgument(_))); + assert_eq!( + crate::error::BlazeDaemonError::from(error).status_code(), + 400 + ); + } + + #[tokio::test] + async fn connect_response_requires_a_numeric_peer_cid() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream + .write_all(b"OK not-a-cid\n") + .await + .expect("invalid peer cid"); + }); + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("invalid peer cid"); + assert!(matches!(error, GuestError::Protocol(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn guest_rejection_is_returned_without_panicking() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": false, "err": "denied by guest"})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .ping() + .await + .expect_err("rejected"); + assert!(matches!(error, GuestError::Rejected(message) if message == "denied by guest")); + } + + #[tokio::test] + async fn read_and_write_response_timeouts_are_bounded() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (stream, _) = accept_request(&listener).await; + tokio::spawn(async move { + let _stream = stream; + tokio::time::sleep(Duration::from_millis(250)).await; + }); + } + }); + let client = GuestClient::new(socket, Duration::from_millis(30), 1024); + let read_error = client + .read_file("/tmp/x".into()) + .await + .expect_err("read timeout"); + assert!(matches!(read_error, GuestError::Timeout(_))); + let write_error = client + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("write timeout"); + assert!(matches!(write_error, GuestError::OutcomeUnknown(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn mutating_handshake_timeout_is_retryable() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let server = tokio::spawn(async move { + let (_stream, _) = listener.accept().await.expect("accept"); + tokio::time::sleep(Duration::from_millis(100)).await; + }); + + let error = GuestClient::new(socket, Duration::from_millis(20), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("handshake timeout"); + assert!(matches!(error, GuestError::Timeout(_))); + server.await.expect("server task"); + } + + #[tokio::test] + async fn mutating_response_corruption_has_unknown_outcome() { + let temp = tempfile::tempdir().expect("temp"); + + let malformed_socket = temp.path().join("malformed.uds"); + let malformed_listener = UnixListener::bind(&malformed_socket).expect("bind"); + tokio::spawn(async move { + let (mut stream, _) = accept_request(&malformed_listener).await; + stream.write_all(b"{not-json}\n").await.expect("response"); + }); + let malformed = GuestClient::new(malformed_socket, Duration::from_secs(15), 1024) + .exec("true".into(), None, None, 1) + .await + .expect_err("malformed response"); + assert!(matches!(malformed, GuestError::OutcomeUnknown(_))); + + let mismatch_socket = temp.path().join("mismatch.uds"); + spawn_server( + mismatch_socket.clone(), + Arc::new(|_| json!({"id": "wrong", "ok": true})), + ) + .await; + let mismatch = GuestClient::new(mismatch_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("mismatched response"); + assert!(matches!(mismatch, GuestError::OutcomeUnknown(_))); + + let missing_ok_socket = temp.path().join("missing-ok.uds"); + spawn_server( + missing_ok_socket.clone(), + Arc::new(|request| json!({"id": request["id"]})), + ) + .await; + let missing_ok = GuestClient::new(missing_ok_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("missing outcome flag"); + assert!(matches!(missing_ok, GuestError::OutcomeUnknown(_))); + + let eof_socket = temp.path().join("eof.uds"); + let eof_listener = UnixListener::bind(&eof_socket).expect("bind"); + tokio::spawn(async move { + let (_stream, _) = accept_request(&eof_listener).await; + }); + let eof = GuestClient::new(eof_socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("response EOF"); + assert!(matches!(eof, GuestError::OutcomeUnknown(_))); + + let oversized_socket = temp.path().join("oversized-write.uds"); + spawn_server( + oversized_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "padding": "xxxxxxxx"})), + ) + .await; + let oversized = GuestClient::new(oversized_socket, Duration::from_secs(1), 1024) + .with_response_limit(4) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("oversized response"); + assert!(matches!(oversized, GuestError::OutcomeUnknown(_))); + } + + #[tokio::test] + async fn explicit_mutating_rejection_has_known_outcome() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("rejected.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": false, "err": "write denied"})), + ) + .await; + + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .write_file("/tmp/x".into(), b"value") + .await + .expect_err("guest rejection"); + assert!(matches!(error, GuestError::Rejected(message) if message == "write denied")); + } + + #[tokio::test] + async fn successful_responses_require_operation_fields() { + let temp = tempfile::tempdir().expect("temp"); + let exec_socket = temp.path().join("exec.uds"); + spawn_server( + exec_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true})), + ) + .await; + let exec = GuestClient::new(exec_socket, Duration::from_secs(15), 1024) + .exec("true".into(), None, None, 1) + .await + .expect_err("missing rc"); + assert!( + matches!(exec, GuestError::OutcomeUnknown(message) if message.contains("missing rc")) + ); + + let read_socket = temp.path().join("read.uds"); + spawn_server( + read_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true})), + ) + .await; + let read = GuestClient::new(read_socket, Duration::from_secs(1), 1024) + .read_file("/tmp/x".into()) + .await + .expect_err("missing data"); + assert!( + matches!(read, GuestError::Protocol(message) if message.contains("missing data_b64")) + ); + } + + #[tokio::test] + async fn exec_inputs_are_bounded_before_connecting() { + let temp = tempfile::tempdir().expect("temp"); + let client = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(15), + 1024, + ); + let command = "x".repeat(MAX_EXEC_COMMAND_BYTES + 1); + assert!(matches!( + client.exec(command, None, None, 1).await, + Err(GuestError::InvalidArgument(_)) + )); + + let mut env = HashMap::new(); + env.insert("KEY".to_string(), "x".repeat(MAX_EXEC_ENV_BYTES)); + assert!(matches!( + client.exec("true".into(), None, Some(env), 1).await, + Err(GuestError::InvalidArgument(_)) + )); + } + + #[tokio::test] + async fn oversized_response_is_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + spawn_server( + socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "data_b64": "AAAAAAAA"})), + ) + .await; + let error = GuestClient::new(socket, Duration::from_secs(1), 1024) + .with_response_limit(4) + .read_file("/tmp/x".into()) + .await + .expect_err("oversized line"); + assert!(matches!(error, GuestError::ResponseTooLarge { .. })); + } + + #[tokio::test] + async fn line_reader_accepts_the_limit_and_rejects_one_more_byte() { + let (mut exact_reader, mut exact_writer) = tokio::io::duplex(64); + tokio::spawn(async move { + exact_writer + .write_all(b"1234\n") + .await + .expect("write exact"); + }); + assert_eq!( + read_line(&mut exact_reader, 4).await.expect("exact limit"), + b"1234" + ); + + let (mut oversized_reader, mut oversized_writer) = tokio::io::duplex(64); + tokio::spawn(async move { + oversized_writer + .write_all(b"12345\n") + .await + .expect("write oversized"); + }); + assert!(matches!( + read_line(&mut oversized_reader, 4) + .await + .expect_err("one byte over"), + GuestError::ResponseTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn invalid_and_decoded_oversized_base64_are_rejected() { + let temp = tempfile::tempdir().expect("temp"); + let invalid_socket = temp.path().join("invalid.uds"); + spawn_server( + invalid_socket.clone(), + Arc::new(|request| json!({"id": request["id"], "ok": true, "data_b64": "not/base64!"})), + ) + .await; + let invalid = GuestClient::new(invalid_socket, Duration::from_secs(1), 16) + .read_file("/tmp/x".into()) + .await + .expect_err("invalid base64"); + assert!(matches!(invalid, GuestError::Protocol(_))); + + let oversized_socket = temp.path().join("oversized.uds"); + spawn_server( + oversized_socket.clone(), + Arc::new(|request| { + json!({ + "id": request["id"], + "ok": true, + "data_b64": BASE64.encode(b"12345") + }) + }), + ) + .await; + let oversized = GuestClient::new(oversized_socket, Duration::from_secs(1), 4) + .read_file("/tmp/x".into()) + .await + .expect_err("decoded limit"); + assert!(matches!( + oversized, + GuestError::ResponseTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn oversized_write_is_rejected_before_connecting() { + let temp = tempfile::tempdir().expect("temp"); + let error = GuestClient::new(temp.path().join("missing.uds"), Duration::from_secs(1), 4) + .write_file("/tmp/x".into(), b"12345") + .await + .expect_err("write limit"); + assert!(matches!( + error, + GuestError::PayloadTooLarge { + actual: 5, + limit: 4 + } + )); + } + + #[tokio::test] + async fn wait_ready_honors_cancellation() { + let temp = tempfile::tempdir().expect("temp"); + let cancellation = CancellationToken::new(); + cancellation.cancel(); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_millis(10), + 1024, + ) + .wait_ready(Duration::from_secs(1), &cancellation) + .await + .expect_err("cancelled"); + assert!(matches!(error, GuestError::Cancelled)); + } + + #[tokio::test] + async fn wait_ready_stops_at_its_deadline() { + let temp = tempfile::tempdir().expect("temp"); + let started = Instant::now(); + let error = GuestClient::new( + temp.path().join("missing.uds"), + Duration::from_secs(1), + 1024, + ) + .wait_ready(Duration::from_millis(60), &CancellationToken::new()) + .await + .expect_err("deadline"); + assert!(matches!(error, GuestError::Timeout(_))); + assert!(started.elapsed() < Duration::from_millis(500)); + } + + #[tokio::test] + async fn wait_ready_retries_after_one_stalled_connection() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("vsock.uds"); + let listener = UnixListener::bind(&socket).expect("bind"); + let attempts = Arc::new(AtomicUsize::new(0)); + let server_attempts = attempts.clone(); + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + break; + }; + let attempt = server_attempts.fetch_add(1, Ordering::Relaxed); + tokio::spawn(async move { + if attempt == 0 { + tokio::time::sleep(Duration::from_secs(1)).await; + return; + } + let connect = read_line(&mut stream, 128).await.expect("connect"); + assert_eq!(connect, b"CONNECT 5000"); + stream.write_all(b"OK 5000\n").await.expect("ok"); + let request = read_line(&mut stream, 4096).await.expect("request"); + let request: serde_json::Value = + serde_json::from_slice(&request).expect("json"); + let response = json!({"id": request["id"], "ok": true}); + let mut bytes = serde_json::to_vec(&response).expect("encode"); + bytes.push(b'\n'); + stream.write_all(&bytes).await.expect("write"); + }); + } + }); + + GuestClient::new(socket, Duration::from_secs(5), 1024) + .wait_ready(Duration::from_secs(1), &CancellationToken::new()) + .await + .expect("second readiness attempt"); + assert!(attempts.load(Ordering::Relaxed) >= 2); + } + + #[test] + fn guest_path_validation_does_not_treat_guest_paths_as_host_paths() { + assert!(validate_guest_path("/tmp/../etc/hosts").is_ok()); + assert!(matches!( + validate_guest_path("relative"), + Err(GuestError::InvalidArgument(_)) + )); + } +} diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index 05a5e8e175..f333027bba 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -15,6 +15,7 @@ mod failpoint; #[path = "failpoint_disabled.rs"] mod failpoint; mod file_provider; +mod guest; mod metrics; mod request_body; mod sandbox; diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 362daedd27..f92384560a 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -15,13 +15,16 @@ use blaze_core::policy::RuntimeDecision; use blaze_core::pool::{PoolKey, PoolManager}; use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; +use tokio_util::sync::CancellationToken; use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; +use crate::guest::{GuestClient, GuestExecResult, MAX_GUEST_FILE_BYTES}; use crate::metrics::Metrics; use crate::spawner::{DynBackendInstance, SpawnerRegistry}; const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); +const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); /// Inputs already parsed and policy-evaluated by the API. #[derive(Debug, Clone)] @@ -82,6 +85,7 @@ pub struct SandboxManager { rootfs_size: u64, mem_size: u64, metrics: Arc, + cancellation: CancellationToken, } /// Construction inputs grouped to keep daemon wiring explicit. @@ -140,6 +144,7 @@ impl SandboxManager { rootfs_size, mem_size, metrics, + cancellation: CancellationToken::new(), }, resources, ) @@ -241,6 +246,40 @@ impl SandboxManager { Ok(ids) } + /// Execute one command through the running sandbox guest. + pub async fn exec( + &self, + id: Uuid, + command: String, + cwd: Option, + env: Option>, + timeout_secs: u32, + ) -> Result { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .exec(command, cwd, env, timeout_secs) + .await + .map_err(BlazeDaemonError::from) + } + + /// Read one file through the running sandbox guest. + pub async fn read_file(&self, id: Uuid, path: String) -> Result> { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .read_file(path) + .await + .map_err(BlazeDaemonError::from) + } + + /// Replace one file through the running sandbox guest. + pub async fn write_file(&self, id: Uuid, path: String, data: &[u8]) -> Result<()> { + let _operation = self.lock_quiescent_state(id, SandboxState::Running).await?; + self.guest_client(id)? + .write_file(path, data) + .await + .map_err(BlazeDaemonError::from) + } + /// Create a cold sandbox or activate a compatible warm runtime. pub async fn create(&self, request: CreateSandbox) -> Result { let pool_key = PoolKey::new( @@ -357,6 +396,20 @@ impl SandboxManager { Ok(backend_instance) => { instance.backend_ownership = BackendOwnership::Running; let actual_backend = backend_instance.backend(); + if let Err(error) = self + .wait_for_guest_ready(&backend_instance, "create-guest-ready") + .await + { + return Err(self + .cleanup_failed_create( + &mut instance, + storage, + Some(backend_instance), + false, + error.into(), + ) + .await); + } let mut backend_instance = Some(backend_instance); let registered = match self.backend_instances.lock() { Ok(mut instances) => { @@ -849,6 +902,11 @@ impl SandboxManager { self.cleanup_owned_instances().await } + /// Cancel readiness polling before the daemon drains active requests. + pub fn begin_shutdown(&self) { + self.cancellation.cancel(); + } + /// Release every lifecycle record and retained backend owner, continuing /// after individual failures. pub async fn cleanup_owned_instances(&self) -> ReconcileReport { @@ -856,6 +914,48 @@ impl SandboxManager { .await } + fn guest_client(&self, id: Uuid) -> Result { + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned() + .ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + let socket = backend.guest_socket_path(); + if socket.as_os_str().is_empty() { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} has no guest transport" + ))); + } + Ok(GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + )) + } + + async fn wait_for_guest_ready( + &self, + backend: &DynBackendInstance, + failpoint: &str, + ) -> crate::guest::Result<()> { + let socket = backend.guest_socket_path(); + if socket.as_os_str().is_empty() { + return Ok(()); + } + crate::failpoint::guest(failpoint)?; + GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + ) + .wait_ready(GUEST_REQUEST_TIMEOUT, &self.cancellation) + .await + } + /// Release owned instances with a caller-supplied per-instance deadline. pub async fn cleanup_owned_instances_with_timeout( &self, diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index f133d91384..e41ed62617 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -15,7 +15,10 @@ use std::time::Instant; use async_trait::async_trait; use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::guest_protocol::DEFAULT_MAX_RESPONSE_BYTES; use blaze_core::{BlazeError, Result}; +use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::UnixListener; use tokio::process::{Child, Command}; use tokio::sync::Mutex; use tokio::task::JoinHandle; @@ -45,6 +48,10 @@ pub struct SpawnResult { pub trait BackendInstance: Send + Sync { /// Concrete backend implementation. fn backend(&self) -> BackendKind; + /// Guest transport endpoint, or an empty path for guestless backends. + fn guest_socket_path(&self) -> &Path { + Path::new("") + } /// Report an observed backend exit without waiting. /// /// `None` means the owned process or task was running when checked. @@ -341,6 +348,7 @@ impl BackendSpawner for MockSpawner { struct MockInstance { instance_id: Uuid, + guest_socket_path: PathBuf, cancellation: CancellationToken, task: Mutex>>, killed: AtomicBool, @@ -348,11 +356,36 @@ struct MockInstance { async fn spawn_mock_instance(instance_id: Uuid, run_dir: PathBuf) -> Result { tokio::fs::create_dir_all(&run_dir).await?; + let socket = run_dir.join("vsock.uds"); + if socket.exists() { + tokio::fs::remove_file(&socket).await?; + } + let listener = UnixListener::bind(&socket)?; let cancellation = CancellationToken::new(); let task_token = cancellation.clone(); - let task = tokio::spawn(async move { task_token.cancelled().await }); + let files = Arc::new(Mutex::new(HashMap::new())); + let task_files = files.clone(); + let task = tokio::spawn(async move { + loop { + tokio::select! { + _ = task_token.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + break; + }; + let files = task_files.clone(); + tokio::spawn(async move { + if let Err(error) = serve_mock_guest(stream, files).await { + tracing::debug!(%error, "mock guest connection ended"); + } + }); + } + } + } + }); Ok(Arc::new(MockInstance { instance_id, + guest_socket_path: socket, cancellation, task: Mutex::new(Some(task)), killed: AtomicBool::new(false), @@ -365,6 +398,10 @@ impl BackendInstance for MockInstance { BackendKind::Mock } + fn guest_socket_path(&self) -> &Path { + &self.guest_socket_path + } + async fn try_wait(&self) -> Result> { let task = { let mut task = self.task.lock().await; @@ -402,11 +439,97 @@ impl BackendInstance for MockInstance { if let Some(task) = task.take() { let _ = task.await; } + if self.guest_socket_path.exists() { + tokio::fs::remove_file(&self.guest_socket_path).await?; + } self.killed.store(true, Ordering::Release); Ok(()) } } +async fn serve_mock_guest( + mut stream: tokio::net::UnixStream, + files: Arc>>>, +) -> std::io::Result<()> { + use base64::Engine; + use base64::engine::general_purpose::STANDARD as BASE64; + + let connect = read_mock_line(&mut stream, 128).await?; + if !connect.starts_with(b"CONNECT ") { + return Ok(()); + } + stream.write_all(b"OK 5000\n").await?; + let request = read_mock_line(&mut stream, DEFAULT_MAX_RESPONSE_BYTES).await?; + let request: serde_json::Value = match serde_json::from_slice(&request) { + Ok(request) => request, + Err(_) => return Ok(()), + }; + let id = request.get("id").cloned().unwrap_or_default(); + let response = match request.get("op").and_then(serde_json::Value::as_str) { + Some("exec") => { + let command = request + .get("cmd") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + serde_json::json!({ + "id": id, + "ok": true, + "rc": 0, + "stdout_b64": BASE64.encode(command.as_bytes()), + "stderr_b64": "" + }) + } + Some("read") => { + let path = request + .get("path") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(); + let data = files.lock().await.get(path).cloned().unwrap_or_default(); + serde_json::json!({"id": id, "ok": true, "data_b64": BASE64.encode(data)}) + } + Some("write") => { + let path = request + .get("path") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_string(); + let data = request + .get("data_b64") + .and_then(serde_json::Value::as_str) + .and_then(|encoded| BASE64.decode(encoded).ok()) + .unwrap_or_default(); + files.lock().await.insert(path, data); + serde_json::json!({"id": id, "ok": true}) + } + _ => serde_json::json!({"id": id, "ok": true}), + }; + let mut encoded = serde_json::to_vec(&response).unwrap_or_else(|_| b"{}".to_vec()); + encoded.push(b'\n'); + stream.write_all(&encoded).await +} + +async fn read_mock_line(stream: &mut R, limit: usize) -> std::io::Result> +where + R: AsyncRead + Unpin, +{ + let mut reader = BufReader::new(stream).take(limit.saturating_add(1) as u64); + let mut output = Vec::with_capacity(limit.min(8192)); + reader.read_until(b'\n', &mut output).await?; + if output.last() == Some(&b'\n') { + output.pop(); + if output.len() <= limit { + return Ok(output); + } + } + if output.len() > limit { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "mock guest line too long", + )); + } + Ok(output) +} + pub(super) async fn terminate_child(child: &mut Child, backend: &str) -> Result<()> { if child.try_wait()?.is_some() { return Ok(()); @@ -820,6 +943,8 @@ mod tests { use blaze_core::policy::BackendConfigs; use blaze_core::storage::StorageSlot; + use crate::guest::GuestClient; + use super::*; fn request(root: &Path) -> SpawnRequest { @@ -865,12 +990,25 @@ mod tests { } #[tokio::test] - async fn mock_instance_reports_liveness_and_supports_idempotent_kill() { + async fn mock_instance_supports_guest_io_and_idempotent_kill() { let temp = tempfile::tempdir().expect("temp"); let instance = MockSpawner .spawn(request(temp.path())) .await .expect("spawn"); + let client = GuestClient::new( + temp.path().join("run/vsock.uds"), + Duration::from_secs(1), + 1024, + ); + client + .write_file("/tmp/value".into(), b"hello") + .await + .expect("write"); + assert_eq!( + client.read_file("/tmp/value".into()).await.expect("read"), + b"hello" + ); assert_eq!(instance.try_wait().await.expect("try wait"), None); instance.kill().await.expect("kill"); assert!(instance.try_wait().await.expect("try wait").is_some()); diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index efac9e95a5..cb25dc9302 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -80,6 +80,7 @@ impl FirecrackerSpawner { child, api_socket, guest_socket, + fc_config.enable_vsock, pid_file, stopped_marker, )); @@ -91,6 +92,7 @@ impl FirecrackerSpawner { child, api_socket, guest_socket, + fc_config.enable_vsock, pid_file, stopped_marker, ); @@ -149,6 +151,7 @@ impl FirecrackerInstance { child: Child, api_socket: PathBuf, guest_socket: PathBuf, + enable_vsock: bool, pid_file: PathBuf, stopped_marker: PathBuf, ) -> Self { @@ -156,7 +159,7 @@ impl FirecrackerInstance { instance_id, child: Mutex::new(Some(child)), api_socket, - guest_socket, + guest_socket: configured_guest_socket(enable_vsock, guest_socket), pid_file, stopped_marker, killed: AtomicBool::new(false), @@ -170,6 +173,10 @@ impl BackendInstance for FirecrackerInstance { BackendKind::Firecracker } + fn guest_socket_path(&self) -> &Path { + &self.guest_socket + } + async fn try_wait(&self) -> Result> { let status = { let mut guard = self.child.lock().await; @@ -393,6 +400,9 @@ async fn wait_for_socket(socket: &Path, child: &mut Child, timeout: Duration) -> } async fn remove_if_exists(path: &Path) -> Result<()> { + if path.as_os_str().is_empty() { + return Ok(()); + } match tokio::fs::remove_file(path).await { Ok(()) => Ok(()), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), @@ -400,6 +410,10 @@ async fn remove_if_exists(path: &Path) -> Result<()> { } } +fn configured_guest_socket(enable_vsock: bool, socket: PathBuf) -> PathBuf { + if enable_vsock { socket } else { PathBuf::new() } +} + fn validate_regular_file(path: &Path, label: &str) -> Result<()> { if !path.is_file() { return Err(BlazeError::BackendError { @@ -528,6 +542,45 @@ mod tests { assert!(value.get("network-interfaces").is_none()); } + #[test] + fn vm_config_and_reported_guest_transport_agree() { + let temp = tempfile::tempdir().expect("temp"); + let request = spawn_request(temp.path()); + let socket = temp.path().join("vsock.uds"); + let disabled = FirecrackerConfig::default(); + let disabled_path = + write_vm_config(&temp.path().join("images"), &request, &disabled, &socket) + .expect("disabled config"); + let disabled_value: serde_json::Value = + serde_json::from_slice(&std::fs::read(disabled_path).expect("read disabled config")) + .expect("parse disabled config"); + assert!(disabled_value.get("vsock").is_none()); + assert!( + configured_guest_socket(disabled.enable_vsock, socket.clone()) + .as_os_str() + .is_empty() + ); + + let enabled = FirecrackerConfig { + enable_vsock: true, + ..FirecrackerConfig::default() + }; + let enabled_path = + write_vm_config(&temp.path().join("images"), &request, &enabled, &socket) + .expect("enabled config"); + let enabled_value: serde_json::Value = + serde_json::from_slice(&std::fs::read(enabled_path).expect("read enabled config")) + .expect("parse enabled config"); + assert_eq!( + enabled_value["vsock"]["uds_path"], + path_string(&socket, "socket").unwrap() + ); + assert_eq!( + configured_guest_socket(enabled.enable_vsock, socket.clone()), + socket + ); + } + #[test] fn serial_log_rotates_before_reuse() { let temp = tempfile::tempdir().expect("temp"); @@ -581,6 +634,7 @@ mod tests { child, temp.path().join("api.sock"), temp.path().join("guest.sock"), + true, pid_file.clone(), stopped_marker(temp.path()), )); From 37e936b91a3fb602f6e6b6a37ddbd4c2b65a131c Mon Sep 17 00:00:00 2001 From: Weisson Date: Thu, 30 Jul 2026 22:20:44 +0800 Subject: [PATCH 21/40] docs(blaze): describe sandbox guest operations The daemon documentation did not explain how callers use the new guest operations or which runtime and size conditions apply. Document the exec, read, and write routes in English and Chinese. Explain cold readiness, lifecycle serialization, warm-pool behavior, retry boundaries, stable error codes, per-request buffering, and the relationship between the default 1 MiB HTTP limit and the 16 MiB decoded file limit. Update Firecracker policy comments to state when guest transport is enabled, and make the quick-start example use a policy that does not require a guest agent. This gives operators enough information to configure and call the feature without assuming unsupported streaming, session reuse, or automatic retries. It does not change runtime behavior. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 69 ++++++++++++++++++++- src/blaze/README_zh.md | 62 +++++++++++++++++- src/blaze/crates/blaze-core/src/policy.rs | 4 +- src/blaze/examples/policies/agent-rl.toml | 2 +- src/blaze/examples/policies/agent-tool.toml | 2 +- 5 files changed, 134 insertions(+), 5 deletions(-) diff --git a/src/blaze/README.md b/src/blaze/README.md index e8988d66f5..69b4a13035 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -15,6 +15,8 @@ Designed as the per-host agent for E2B-style orchestrator platforms. - **Policy-driven backend selection** — workload class → backend priority list - **Lifecycle state machine** — 9 states: Pending, Creating, Running, Paused, Checkpointed, RecoveryRequired, Reset, Warm, and Destroyed +- **Guest operations** — bounded command execution and file transfer for + running backends that expose a guest endpoint - **Warm pool management** — pre-warmed instances with TTL-based GC - **Template registry** — in-memory template tracking with idle eviction - **Kernel hook registry** — state tracking for pre/post hooks @@ -41,9 +43,13 @@ curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # Create a sandbox curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ - -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' ``` +The quick-start request uses an example policy with Firecracker guest transport +disabled, so an image without the compatible guest agent does not wait for guest +readiness. Enable the transport only for images that run that agent. + ## Configuration The daemon reads a TOML config file (default: `/etc/anolisa/blaze/config.toml`) @@ -71,6 +77,18 @@ byte count: max_body_bytes = 1048576 ``` +Guest files are limited to 16 MiB after base64 decoding. A full-size write is +larger on the wire because JSON and base64 add overhead, so the default 1 MiB +request limit intentionally rejects it. Set at least 22 MiB when callers need +the full decoded limit: + +```toml +[api] +max_body_bytes = 23068672 +``` + +The daemon checks both the HTTP request size and the decoded file size. + ### VM Resource Configuration Blaze resolves vCPU and memory settings using a three-layer fallback chain: @@ -117,11 +135,17 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/sandboxes` | Create a sandbox | | GET | `/v1/sandboxes/{id}` | Get sandbox details | | DELETE | `/v1/sandboxes/{id}` | Destroy a sandbox | +| POST | `/v1/sandboxes/{id}/exec` | Execute a guest command | +| POST | `/v1/sandboxes/{id}/read` | Read a guest file | +| POST | `/v1/sandboxes/{id}/write` | Replace a guest file | | GET | `/v1/instances` | Alias for listing sandboxes | | POST | `/v1/instances` | Alias for creating a sandbox | | GET | `/v1/instances/{id}` | Alias for sandbox details | | DELETE | `/v1/instances/{id}` | Alias for destroying a sandbox | | POST | `/v1/instances/{id}/destroy` | Compatible destroy action | +| POST | `/v1/instances/{id}/exec` | Compatible guest command action | +| POST | `/v1/instances/{id}/read` | Compatible guest file read action | +| POST | `/v1/instances/{id}/write` | Compatible guest file write action | | POST | `/v1/instances/{id}/checkpoint` | Reserved; returns `501` until backend and storage capture is implemented | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | GET | `/v1/pools` | List warm pools | @@ -160,6 +184,49 @@ sandbox is running with no active lifecycle operation, then return `501` without changing runtime or persisted state. Their backend and storage operations are not implemented here. +### Guest operations + +Guest operations are available only while a sandbox is `Running` and its +backend reports a guest endpoint. A cold create that reports such an endpoint +waits for the guest agent to answer before publishing `Running`. Backends with +guest support disabled skip that wait, and later guest-operation requests +return HTTP 409. Warm-pool activation does not currently repeat the guest +readiness probe. + +Guest operations and lifecycle changes use the same per-sandbox operation +lock. A request may wait for an earlier lifecycle action. After it obtains the +lock, the manager checks `Running` again; if destroy or another state change +won the race, the guest request fails without contacting the old runtime. + +The endpoints accept JSON: + +```json +{"cmd":"uname -a","cwd":"/","env":{"LANG":"C"},"timeout":10} +``` + +```json +{"path":"/tmp/input","data_b64":"aGVsbG8="} +``` + +`read` takes only `path`; successful file reads and command output use standard +base64 in the response. Exec timeouts must be from 1 through 20 seconds. Guest +files are limited to 16 MiB after decoding, and response frames are bounded. + +An exec or write failure before request delivery is an ordinary transport +failure. A bounded wait that expires before delivery uses +`"code": "guest_timeout"`. If delivery began but the daemon cannot determine +the result, the API returns HTTP 504 with +`"code": "guest_outcome_unknown"`; callers must reconcile state instead of +automatically replaying the operation. Reads do not change guest state and +remain safe for caller-directed retry. An oversized read response returns +HTTP 502 with `"code": "guest_response_too_large"`. For exec or write after +delivery starts, an oversized or otherwise untrusted response instead leaves +the outcome unknown. An oversized caller request returns HTTP 413. + +Each request is fully buffered. The limits bound one request, not the sum of +concurrent requests, so callers should also bound guest-operation concurrency. +Streaming files, interactive terminals, and session reuse are not supported. + #### Health Check `GET /v1/health` returns daemon status including storage pool readiness: diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index ff96fa68e7..df9d643e3e 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -14,6 +14,7 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **策略驱动后端选择** — workload class → 后端优先级列表 - **生命周期状态机** — 9 种状态：Pending、Creating、Running、Paused、 Checkpointed、RecoveryRequired、Reset、Warm 和 Destroyed +- **Guest 操作** — 对提供 guest endpoint 的运行中后端执行有界命令和文件传输 - **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC - **模板注册表** — 内存中模板追踪，支持空闲驱逐 - **内核 hook 注册** — 前/后置 hook 状态追踪 @@ -40,9 +41,13 @@ curl --unix-socket /run/blaze/api.sock http://localhost/v1/health # 创建 sandbox curl -X POST --unix-socket /run/blaze/api.sock http://localhost/v1/sandboxes \ -H 'Content-Type: application/json' \ - -d '{"workload_class":"agent-rl","image_digest":"sha256:..."}' + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' ``` +快速开始使用关闭 Firecracker guest transport 的示例策略，因此没有兼容 +guest agent 的镜像不会等待 guest 就绪。只有镜像运行了对应 agent 时才应 +启用该 transport。 + ## 配置 daemon 读取 TOML 配置文件（默认：`/etc/anolisa/blaze/config.toml`） @@ -69,6 +74,17 @@ daemon 默认接收不超过 1 MiB 的请求体。它会同时检查声明的 max_body_bytes = 1048576 ``` +Guest 文件在 base64 解码后最多为 16 MiB。完整的 16 MiB 写入经过 JSON 和 +base64 编码后会变大，所以默认 1 MiB 请求上限会拒绝它。调用者确实需要完整 +文件上限时，应至少配置 22 MiB： + +```toml +[api] +max_body_bytes = 23068672 +``` + +daemon 会同时检查 HTTP 请求大小和解码后的文件大小。 + ### VM 资源配置 Blaze 使用三层回退链解析 vCPU 和内存设置： @@ -115,11 +131,17 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/sandboxes` | 创建 sandbox | | GET | `/v1/sandboxes/{id}` | 获取 sandbox 详情 | | DELETE | `/v1/sandboxes/{id}` | 销毁 sandbox | +| POST | `/v1/sandboxes/{id}/exec` | 执行 guest 命令 | +| POST | `/v1/sandboxes/{id}/read` | 读取 guest 文件 | +| POST | `/v1/sandboxes/{id}/write` | 替换 guest 文件 | | GET | `/v1/instances` | 列出 sandbox 的兼容入口 | | POST | `/v1/instances` | 创建 sandbox 的兼容入口 | | GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | | DELETE | `/v1/instances/{id}` | 销毁 sandbox 的兼容入口 | | POST | `/v1/instances/{id}/destroy` | 保留的销毁 action | +| POST | `/v1/instances/{id}/exec` | Guest 命令兼容入口 | +| POST | `/v1/instances/{id}/read` | Guest 文件读取兼容入口 | +| POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | | POST | `/v1/instances/{id}/checkpoint` | 预留接口；后端和存储快照实现前返回 `501` | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | GET | `/v1/pools` | 列出 warm pool | @@ -153,6 +175,44 @@ daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失 `Running` 且没有进行中的生命周期操作，然后返回 `501`；它们不会修改运行 资源或持久化状态，其后端和存储操作尚未在这里实现。 +### Guest 操作 + +只有 sandbox 处于 `Running` 且后端报告了 guest endpoint 时，才能执行 +guest 操作。冷启动后端如果报告了该 endpoint，创建流程会等待 guest agent +响应后才发布 `Running`。关闭 guest 支持的后端会跳过等待，后续 guest +操作返回 HTTP 409。当前从 warm pool 激活实例时不会再次执行 guest +readiness 探测。 + +Guest 操作和生命周期变更使用同一个 sandbox 操作锁。请求可能等待先开始的 +生命周期操作；取得锁后，manager 会再次检查 `Running`。如果 destroy 或 +其他状态变更先完成，guest 请求不会访问旧 runtime，而是直接失败。 + +接口接收以下 JSON： + +```json +{"cmd":"uname -a","cwd":"/","env":{"LANG":"C"},"timeout":10} +``` + +```json +{"path":"/tmp/input","data_b64":"aGVsbG8="} +``` + +`read` 只需要 `path`；文件读取结果和命令输出使用标准 base64。Exec timeout +范围为 1 至 20 秒。Guest 文件解码后最多为 16 MiB，响应帧也有固定上限。 + +如果 exec 或 write 在请求送达前失败，它是普通通信失败；送达前等待超时使用 +`"code": "guest_timeout"`。如果已经开始送达，但 daemon 无法确定结果，API +返回 HTTP 504 和 `"code": "guest_outcome_unknown"`；调用者应先核对状态， +不能自动重放。read 不改变 guest 状态，可以由调用者决定是否重试。Guest +read 返回内容过大时，API 返回 HTTP 502 和 +`"code": "guest_response_too_large"`。exec 或 write 已经开始送达后，如果 +返回内容过大或不可信，结果仍归为 unknown。调用者的请求过大时返回 HTTP +413。 + +每个请求都会完整缓冲。上限约束的是单个请求，而不是所有并发请求之和，因此 +调用方还需要限制 guest 操作并发数。当前不支持文件流式传输、交互式终端和 +会话复用。 + #### 健康检查 `GET /v1/health` 返回 daemon 状态，包含存储池就绪信息： diff --git a/src/blaze/crates/blaze-core/src/policy.rs b/src/blaze/crates/blaze-core/src/policy.rs index bfcbac3aa8..ff823c3895 100644 --- a/src/blaze/crates/blaze-core/src/policy.rs +++ b/src/blaze/crates/blaze-core/src/policy.rs @@ -358,7 +358,9 @@ pub struct FirecrackerConfig { /// Guest kernel command line. #[serde(default = "default_fc_boot_args")] pub boot_args: String, - /// Enable virtio-vsock (Phase 2+; parsed today but not yet wired). + /// Enable guest operations over virtio-vsock. + /// + /// This is disabled by default and requires a compatible guest service. #[serde(default)] pub enable_vsock: bool, /// Capture guest ttyS0 output (Firecracker stdout) to `serial.log`. diff --git a/src/blaze/examples/policies/agent-rl.toml b/src/blaze/examples/policies/agent-rl.toml index 8fe5e79695..deaa5be3fd 100644 --- a/src/blaze/examples/policies/agent-rl.toml +++ b/src/blaze/examples/policies/agent-rl.toml @@ -117,7 +117,7 @@ memory = "4G" # --------------------------------------------------------------------------- [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" -enable_vsock = true # parsed but not yet wired (Phase 2+) +enable_vsock = true # requires the compatible guest agent on vsock port 5000 serial_log = false # --------------------------------------------------------------------------- diff --git a/src/blaze/examples/policies/agent-tool.toml b/src/blaze/examples/policies/agent-tool.toml index 1d77a9b410..ed6df07571 100644 --- a/src/blaze/examples/policies/agent-tool.toml +++ b/src/blaze/examples/policies/agent-tool.toml @@ -105,7 +105,7 @@ memory = "1G" # --------------------------------------------------------------------------- [backend.firecracker] boot_args = "console=ttyS0 reboot=k panic=1 pci=off" -enable_vsock = false # parsed but not yet wired (Phase 2+) +enable_vsock = false # enable only when the image runs the compatible guest agent memory = "2G" vcpus = 2 serial_log = false From 76b078aa3c3e02258369b60e5fdeb08c1f24d0fb Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 04:07:19 +0800 Subject: [PATCH 22/40] feat(blaze): inventory owned file slots Runtime cleanup after restart needs a complete view of provider-owned slots, including directories left before all artifacts were created. The storage contract previously exposed release by ID but did not declare whether a provider could enumerate every owned ID. Add an explicit capability for complete inventory under the currently configured provider root. The file provider returns validated IDs in stable order and refuses entries that are not direct slot directories. Keep release-by-ID idempotent for absent or partial directories and reject non-directory aliases. This gives later runtime cleanup a provider-level foundation without recording or selecting a provider or root. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/storage.rs | 22 +++ src/blaze/crates/blazed/src/file_provider.rs | 156 ++++++++++++++++++- 2 files changed, 172 insertions(+), 6 deletions(-) diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index aa4961a6ca..794f756682 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -119,6 +119,28 @@ pub trait StorageProvider: Send + Sync { self.release(slot).await } + /// Report whether pool-owned slots can be inventoried and released by ID. + /// + /// Returning true promises both a complete [`Self::list_owned_ids`] + /// inventory for the provider's currently configured root and an + /// idempotent [`Self::release_by_id`] that can retry complete, missing, + /// and partially created slots. It does not identify a provider or root + /// recorded by another subsystem. + fn supports_runtime_pool_recovery(&self) -> bool { + false + } + + /// List every stable slot identifier currently owned by the provider. + /// + /// Implementations must reject entries that cannot be classified as a + /// provider-owned slot. Callers use this inventory only when + /// [`Self::supports_runtime_pool_recovery`] returns true. + async fn list_owned_ids(&self) -> Result> { + Err(BlazeError::StorageError { + msg: "storage provider does not expose stable slot inventory".to_string(), + }) + } + /// Reconstruct a previously allocated slot from a stable instance id. /// /// Implementations must derive every returned path from their configured diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index fb78fdb67b..c82f7c570e 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -200,12 +200,29 @@ impl StorageProvider for FileStorageProvider { // Re-derive the canonical path from instances_dir + slot.id. Do not // trust path strings carried in a persisted or externally built slot. let canonical_dir = self.slot_for_id(&slot.id)?.instance_dir; - if canonical_dir.exists() { - tokio::fs::remove_dir_all(&canonical_dir) - .await - .map_err(|e| BlazeError::StorageError { - msg: format!("release '{}': {}", slot.id, e), - })?; + match tokio::fs::symlink_metadata(&canonical_dir).await { + Ok(metadata) if metadata.file_type().is_dir() => { + tokio::fs::remove_dir_all(&canonical_dir) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("release '{}': {error}", slot.id), + })?; + } + Ok(_) => { + return Err(BlazeError::StorageError { + msg: format!( + "release '{}': refusing non-directory slot {}", + slot.id, + canonical_dir.display() + ), + }); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(BlazeError::StorageError { + msg: format!("release '{}': inspect: {error}", slot.id), + }); + } } Ok(()) } @@ -215,6 +232,53 @@ impl StorageProvider for FileStorageProvider { self.release(slot).await } + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> Result> { + let mut entries = tokio::fs::read_dir(&self.instances_dir) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: {error}", self.instances_dir.display()), + })?; + let mut ids = Vec::new(); + while let Some(entry) = + entries + .next_entry() + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: {error}", self.instances_dir.display()), + })? + { + let path = entry.path(); + let file_type = entry + .file_type() + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inventory {}: inspect: {error}", path.display()), + })?; + if !file_type.is_dir() { + return Err(BlazeError::StorageError { + msg: format!( + "inventory {}: unexpected non-directory entry", + path.display() + ), + }); + } + let id = entry + .file_name() + .into_string() + .map_err(|_| BlazeError::StorageError { + msg: format!("inventory {}: slot name is not UTF-8", path.display()), + })?; + validate_instance_id(&id)?; + ids.push(id); + } + ids.sort(); + Ok(ids) + } + async fn reconstruct(&self, instance_id: &str) -> Result { let slot = self.slot_for_id(instance_id)?; require_slot_path(instance_id, &slot.instance_dir, RequiredPathType::Directory).await?; @@ -326,6 +390,7 @@ fn validate_instance_id(instance_id: &str) -> Result<()> { #[cfg(test)] mod tests { use super::*; + use uuid::Uuid; #[tokio::test] async fn probe_existing_dir_returns_true() { @@ -382,6 +447,85 @@ mod tests { assert!(!dir.exists()); } + #[tokio::test] + async fn release_by_id_recovers_missing_and_partial_slots() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let id = Uuid::new_v4().to_string(); + let missing_id = Uuid::new_v4().to_string(); + provider.release_by_id(&missing_id).await.unwrap(); + provider.release_by_id(&missing_id).await.unwrap(); + let partial = tmp.path().join(&id); + tokio::fs::create_dir(&partial).await.unwrap(); + tokio::fs::write(partial.join("rootfs.ext4"), b"partial") + .await + .unwrap(); + + provider.release_by_id(&id).await.unwrap(); + provider.release_by_id(&id).await.unwrap(); + + assert!(!partial.exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn release_by_id_rejects_non_directory_and_symlink_slots() { + use std::os::unix::fs::symlink; + + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let id = Uuid::new_v4().to_string(); + let slot_path = tmp.path().join(&id); + tokio::fs::write(&slot_path, b"not a directory") + .await + .unwrap(); + + let file_error = provider.release_by_id(&id).await.unwrap_err(); + assert!(file_error.to_string().contains("refusing non-directory")); + assert!(slot_path.is_file()); + + tokio::fs::remove_file(&slot_path).await.unwrap(); + let target = tempfile::TempDir::new().unwrap(); + symlink(target.path(), &slot_path).unwrap(); + + let symlink_error = provider.release_by_id(&id).await.unwrap_err(); + assert!(symlink_error.to_string().contains("refusing non-directory")); + assert!(std::fs::symlink_metadata(&slot_path).unwrap().is_symlink()); + assert!(target.path().is_dir()); + } + + #[tokio::test] + async fn owned_slot_inventory_returns_stable_ids() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + let first = Uuid::new_v4().to_string(); + let second = Uuid::new_v4().to_string(); + tokio::fs::create_dir(tmp.path().join(&second)) + .await + .unwrap(); + tokio::fs::create_dir(tmp.path().join(&first)) + .await + .unwrap(); + let mut expected = vec![first, second]; + expected.sort(); + + assert!(provider.supports_runtime_pool_recovery()); + assert_eq!(provider.list_owned_ids().await.unwrap(), expected); + } + + #[tokio::test] + async fn owned_slot_inventory_rejects_unknown_entry_types() { + let tmp = tempfile::TempDir::new().unwrap(); + let provider = FileStorageProvider::new(tmp.path().to_path_buf()); + tokio::fs::write(tmp.path().join("unexpected"), b"not a slot") + .await + .unwrap(); + + let error = provider.list_owned_ids().await.unwrap_err(); + + assert!(error.to_string().contains("unexpected non-directory")); + } + #[tokio::test] async fn pool_status_returns_defaults() { let tmp = tempfile::TempDir::new().unwrap(); From 0b461e18c601e208e498c108a8466aa84cd9f8d8 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 05:30:15 +0800 Subject: [PATCH 23/40] feat(blaze): refill warm runtime slots POST /v1/sandboxes can now claim a compatible runtime slot after the first eligible create configures bounded background construction. Each slot owns independent storage; prefork mode also starts and checks the selected backend before the slot becomes ready. Persist pool ownership before allocation and transfer it to lifecycle state with a token so create, destroy, startup reconciliation, and daemon shutdown agree on one owner. Retain incomplete builds, handoffs, and cleanup work for retry instead of returning them to the ready queue. The target bounds pool-owned and in-flight work, not sandboxes after handoff. The first accepted build shape remains fixed for this daemon run; incompatible requests use the normal create path. Restart cleans unclaimed slots instead of restoring the in-memory ready queue. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/config.rs | 130 +- src/blaze/crates/blaze-core/src/lifecycle.rs | 180 + src/blaze/crates/blaze-core/src/policy.rs | 40 +- src/blaze/crates/blazed/src/api.rs | 571 ++- src/blaze/crates/blazed/src/daemon.rs | 64 +- src/blaze/crates/blazed/src/main.rs | 1 + src/blaze/crates/blazed/src/runtime_pool.rs | 13 + .../crates/blazed/src/runtime_pool/pool.rs | 2451 +++++++++++ .../blazed/src/runtime_pool/recovery.rs | 3660 +++++++++++++++++ .../crates/blazed/src/sandbox/manager.rs | 774 +++- src/blaze/crates/blazed/src/spawner.rs | 4 + src/blaze/crates/blazed/src/state.rs | 22 +- src/blaze/examples/config.toml | 18 +- 13 files changed, 7841 insertions(+), 87 deletions(-) create mode 100644 src/blaze/crates/blazed/src/runtime_pool.rs create mode 100644 src/blaze/crates/blazed/src/runtime_pool/pool.rs create mode 100644 src/blaze/crates/blazed/src/runtime_pool/recovery.rs diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index 2c490167a2..64336a701a 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -8,6 +8,7 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; use crate::error::{BlazeError, ConfigErrorSource, Result}; +use crate::policy::parse_duration; /// Top-level daemon configuration. #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -168,13 +169,11 @@ pub struct StorageSection { #[serde(default = "default_storage_provider")] pub provider: String, - /// Warm pool target size (0 = no pool). - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Warm runtime target size (0 disables background construction). #[serde(default)] pub pool_size: usize, - /// Whether to pre-start VMs in pool slots. - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Whether to pre-start backends in warm runtime slots. #[serde(default)] pub prefork: bool, @@ -226,16 +225,30 @@ impl DaemonConfig { ), }); } - validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir) + for (name, value) in [ + ("pool.default_warm_ttl", self.pool.default_warm_ttl.as_str()), + ("pool.gc_interval", self.pool.gc_interval.as_str()), + ] { + if parse_duration(value).is_none() { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "{name} must be a positive duration with an s, m, h, or d suffix, got \ + {value:?}" + )), + }); + } + } + validate_runtime_storage_paths( + &self.daemon.state_dir.join("runtime-pool"), + &self.storage.images_dir, + &self.storage.instances_dir, + ) } } /// Reject storage roots whose ownership domains overlap. pub fn validate_storage_paths(images_dir: &Path, instances_dir: &Path) -> Result<()> { - if images_dir == instances_dir - || images_dir.starts_with(instances_dir) - || instances_dir.starts_with(images_dir) - { + if paths_overlap(images_dir, instances_dir) { return Err(BlazeError::ConfigError { source: ConfigErrorSource::InvalidValue(format!( "storage.images_dir ({}) and storage.instances_dir ({}) must be disjoint", @@ -247,6 +260,38 @@ pub fn validate_storage_paths(images_dir: &Path, instances_dir: &Path) -> Result Ok(()) } +/// Reject runtime and storage roots whose ownership domains overlap. +/// +/// Callers should invoke this once with configured paths and again with +/// canonical paths after creating the roots. The second check catches aliases +/// introduced by symbolic links. +pub fn validate_runtime_storage_paths( + runtime_root: &Path, + images_dir: &Path, + instances_dir: &Path, +) -> Result<()> { + validate_storage_paths(images_dir, instances_dir)?; + for (name, storage_root) in [ + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ] { + if paths_overlap(runtime_root, storage_root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime slot root ({}) and {name} ({}) must be disjoint", + runtime_root.display(), + storage_root.display() + )), + }); + } + } + Ok(()) +} + +fn paths_overlap(left: &Path, right: &Path) -> bool { + left == right || left.starts_with(right) || right.starts_with(left) +} + // ----- defaults ----- fn default_log_level() -> String { @@ -370,6 +415,28 @@ mod tests { ); } + #[test] + fn rejects_invalid_runtime_pool_durations() { + for (field, value) in [ + ("pool.default_warm_ttl", "30"), + ("pool.default_warm_ttl", "0s"), + ("pool.gc_interval", "soon"), + ("pool.gc_interval", "0m"), + ] { + let mut cfg = DaemonConfig::default(); + if field == "pool.default_warm_ttl" { + cfg.pool.default_warm_ttl = value.to_string(); + } else { + cfg.pool.gc_interval = value.to_string(); + } + + let error = cfg.validate().expect_err("invalid pool duration"); + + assert!(error.to_string().contains(field)); + assert!(error.to_string().contains(value)); + } + } + #[test] fn rejects_equal_or_nested_storage_roots() { for (images, instances) in [ @@ -384,4 +451,49 @@ mod tests { assert!(error.to_string().contains("must be disjoint")); } } + + #[test] + fn accepts_sibling_runtime_and_storage_roots() { + validate_runtime_storage_paths( + Path::new("/var/lib/blaze/runtime-pool"), + Path::new("/var/lib/blaze/images"), + Path::new("/var/lib/blaze/instances"), + ) + .expect("sibling ownership roots are disjoint"); + } + + #[test] + fn rejects_equal_or_nested_runtime_and_storage_roots() { + for (runtime, images, instances) in [ + ( + "/var/lib/blaze/images", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze/images/runtime", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ( + "/var/lib/blaze/instances/runtime", + "/var/lib/blaze/images", + "/var/lib/blaze/instances", + ), + ] { + let error = validate_runtime_storage_paths( + Path::new(runtime), + Path::new(images), + Path::new(instances), + ) + .expect_err("overlapping runtime and storage roots"); + assert!(matches!(error, BlazeError::ConfigError { .. })); + assert!(error.to_string().contains("must be disjoint")); + } + } } diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index c270e546d8..cf9611a232 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -93,6 +93,20 @@ pub enum StartPath { Warm, } +/// Canonical directory family that owns backend runtime artifacts. +/// +/// This is independent from [`StartPath`]: a sandbox can be activated through +/// a warm lifecycle transition while retaining its original sandbox directory. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RuntimeLocation { + /// Runtime artifacts live under the sandbox's lifecycle state directory. + #[default] + Sandbox, + /// Runtime artifacts live under the daemon's warm-slot directory. + WarmPool, +} + /// Durable knowledge about whether a backend may still own a live process. /// /// `Unknown` is the safe default for state written by older daemon versions. @@ -119,6 +133,12 @@ pub struct SandboxInstance { pub created_at: DateTime, pub updated_at: DateTime, pub policy_name: String, + /// Canonical directory family containing backend runtime artifacts. + #[serde(default)] + pub runtime_location: RuntimeLocation, + /// Stable nonce that links a claimed warm runtime to its slot journal. + #[serde(default)] + pub runtime_owner_token: Option, /// Last durably known backend ownership state. #[serde(default)] pub backend_ownership: BackendOwnership, @@ -149,11 +169,50 @@ impl SandboxInstance { created_at: now, updated_at: now, policy_name, + runtime_location: RuntimeLocation::Sandbox, + runtime_owner_token: None, backend_ownership: BackendOwnership::NotStarted, operation: None, } } + /// Adopt a ready runtime slot into a recoverable create operation. + pub fn new_warm_claim( + id: Uuid, + backend: BackendKind, + workload_class: WorkloadClass, + image_digest: String, + policy_name: String, + backend_ownership: BackendOwnership, + runtime_owner_token: Uuid, + ) -> Result { + if !matches!( + backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Running + ) { + return Err(BlazeError::BackendError { + msg: format!( + "ready runtime slot {id} has invalid backend ownership {backend_ownership:?}" + ), + }); + } + let mut instance = Self::new( + backend, + workload_class, + image_digest, + StartPath::Warm, + policy_name, + ); + instance.id = id; + instance.state = SandboxState::Warm; + instance.runtime_location = RuntimeLocation::WarmPool; + instance.runtime_owner_token = Some(runtime_owner_token); + instance.backend_ownership = backend_ownership; + instance.begin_operation(OperationKind::Create)?; + instance.transition(SandboxState::Creating)?; + Ok(instance) + } + /// Record a new operation before starting its first owned-resource /// mutation. An unfinished journal must be recovered rather than silently /// replaced by a later request. @@ -193,6 +252,16 @@ impl SandboxInstance { self.updated_at = Utc::now(); } + /// Return whether lifecycle metadata proves that no runtime owner remains. + pub fn is_clean_terminal(&self) -> bool { + self.state == SandboxState::Destroyed + && self.operation.is_none() + && matches!( + self.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) + } + /// Apply a state transition. Returns /// [`BlazeError::InvalidStateTransition`] when the move is not part /// of the lifecycle state graph. @@ -394,6 +463,39 @@ mod tests { )); } + #[test] + fn clean_terminal_requires_safe_ownership_and_no_operation() { + let mut destroyed = fresh(); + destroyed + .transition(SandboxState::Destroyed) + .expect("destroyed"); + for ownership in [BackendOwnership::NotStarted, BackendOwnership::Stopped] { + destroyed.backend_ownership = ownership; + assert!(destroyed.is_clean_terminal()); + } + + destroyed.backend_ownership = BackendOwnership::Running; + assert!(!destroyed.is_clean_terminal()); + + let mut unfinished = fresh(); + unfinished + .begin_operation(OperationKind::Create) + .expect("begin create"); + unfinished + .transition(SandboxState::Destroyed) + .expect("destroyed"); + unfinished.backend_ownership = BackendOwnership::Stopped; + assert!(!unfinished.is_clean_terminal()); + + let mut running = fresh(); + running + .transition(SandboxState::Creating) + .expect("creating"); + running.transition(SandboxState::Running).expect("running"); + running.backend_ownership = BackendOwnership::Stopped; + assert!(!running.is_clean_terminal()); + } + #[test] fn recovery_required_can_finish_but_cannot_be_reentered() { let mut inst = fresh(); @@ -577,6 +679,84 @@ mod tests { let loaded: SandboxInstance = serde_json::from_value(value).expect("legacy state"); assert!(loaded.operation.is_none()); assert_eq!(loaded.backend_ownership, BackendOwnership::Unknown); + assert_eq!(loaded.runtime_location, RuntimeLocation::Sandbox); + assert!(loaded.runtime_owner_token.is_none()); + } + + #[test] + fn warm_start_classification_does_not_move_runtime_artifacts() { + let mut instance = fresh(); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.transition(SandboxState::Reset).expect("reset"); + instance.transition(SandboxState::Warm).expect("warm"); + instance + .transition(SandboxState::Creating) + .expect("warm creating"); + + assert_eq!(instance.start_path, StartPath::Warm); + assert_eq!(instance.runtime_location, RuntimeLocation::Sandbox); + } + + #[test] + fn runtime_location_round_trips() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + let owner_token = Uuid::new_v4(); + instance.runtime_location = RuntimeLocation::WarmPool; + instance.runtime_owner_token = Some(owner_token); + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + + assert_eq!(loaded.runtime_location, RuntimeLocation::WarmPool); + assert_eq!(loaded.runtime_owner_token, Some(owner_token)); + } + + #[test] + fn warm_claim_starts_one_durable_create_operation() { + let id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + + let instance = SandboxInstance::new_warm_claim( + id, + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:warm".into(), + "warm-policy".into(), + BackendOwnership::Running, + owner_token, + ) + .expect("warm claim"); + + assert_eq!(instance.id, id); + assert_eq!(instance.state, SandboxState::Creating); + assert_eq!(instance.start_path, StartPath::Warm); + assert_eq!(instance.runtime_location, RuntimeLocation::WarmPool); + assert_eq!(instance.runtime_owner_token, Some(owner_token)); + assert_eq!(instance.backend_ownership, BackendOwnership::Running); + assert_eq!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Create) + ); + } + + #[test] + fn warm_claim_rejects_an_unstable_backend_owner() { + let error = SandboxInstance::new_warm_claim( + Uuid::new_v4(), + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:warm".into(), + "warm-policy".into(), + BackendOwnership::Starting, + Uuid::new_v4(), + ) + .expect_err("starting backend cannot be claimed"); + + assert!(matches!(error, BlazeError::BackendError { .. })); } #[test] diff --git a/src/blaze/crates/blaze-core/src/policy.rs b/src/blaze/crates/blaze-core/src/policy.rs index ff823c3895..63f5ffd241 100644 --- a/src/blaze/crates/blaze-core/src/policy.rs +++ b/src/blaze/crates/blaze-core/src/policy.rs @@ -184,15 +184,15 @@ impl PolicyFile { }); } - // Validate [pool].warm_ttl format (e.g. "30s", "30m", "1h", "1d"; pure numbers are illegal). + // Validate an explicit pool TTL while allowing the daemon default to apply when omitted. if let Some(pool) = self.pool.as_ref() - && parse_duration(&pool.warm_ttl).is_none() + && let Some(warm_ttl) = pool.warm_ttl.as_deref() + && parse_duration(warm_ttl).is_none() { return Err(BlazeError::PolicyEvalError { reason: format!( "policy \"{policy_name}\": [pool].warm_ttl must be a duration like \"30s\", \"30m\", \"1h\", \"1d\", got \"{warm_ttl}\"", policy_name = self.policy_name, - warm_ttl = pool.warm_ttl ), }); } @@ -297,16 +297,13 @@ pub struct PolicyPool { pub target: u32, #[serde(default)] pub max: u32, - #[serde(default = "default_warm_ttl")] - pub warm_ttl: String, + /// Optional TTL override; the daemon-wide pool default applies when absent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub warm_ttl: Option, #[serde(default)] pub reset_mode: ResetMode, } -fn default_warm_ttl() -> String { - "30m".to_string() -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PolicyCheckpoint { #[serde(default)] @@ -833,6 +830,31 @@ sequence = ["template-reg:bind-mm-template"] assert_eq!(pf.match_.workload_class, WorkloadClass::AgentRl); assert_eq!(pf.select.backend_priority[0], BackendKind::KataFc); assert!(pf.pool.as_ref().expect("pool").enabled); + assert_eq!( + pf.pool.as_ref().and_then(|pool| pool.warm_ttl.as_deref()), + Some("30m") + ); + } + + #[test] + fn omitted_pool_ttl_remains_unset_for_daemon_resolution() { + let raw = r#" +manifest_version = 1 +policy_name = "test" + +[match] +workload_class = "agent-tool" + +[select] +backend_priority = ["mock"] + +[pool] +enabled = true +"#; + let policy: PolicyFile = toml::from_str(raw).expect("parse"); + + policy.validate().expect("omitted TTL is valid"); + assert!(policy.pool.as_ref().expect("pool").warm_ttl.is_none()); } #[test] diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index fe77d055c2..2bc38ad7fb 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -221,7 +221,7 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result, body: &[u8]) -> Result = { @@ -475,7 +487,8 @@ fn decode_guest_file(encoded: &str, limit: usize) -> Result> { /// Cleanup starts concurrently for all known owners and shares one deadline. /// This lets independent owners finish after another owner fails or stalls /// without multiplying the daemon's shutdown time by the sandbox count. -/// Timed-out tasks are cancelled and joined before this function returns. +/// Timed-out sandbox tasks are cancelled and joined. Runtime-pool shutdown +/// retains control of its nested worker and joins that worker itself. pub(crate) async fn shutdown_instances(state: &Arc, budget: Duration) -> Result<()> { let ids = state.manager.owned_instance_ids()?; @@ -485,33 +498,52 @@ pub(crate) async fn shutdown_instances(state: &Arc, budget: Duratio for id in ids { let state = state.clone(); let task = tasks.spawn(async move { state.manager.destroy(id).await.map(|_| ()) }); - task_owners.insert(task.id(), id); + task_owners.insert(task.id(), id.to_string()); } + let pool_state = state.clone(); + let mut pool_shutdown = Box::pin(async move { + pool_state + .manager + .shutdown_runtime_pool_until(deadline) + .await + }); + let mut pool_pending = true; + let mut deadline_sleep = Box::pin(tokio::time::sleep_until(deadline)); let mut failures = BTreeMap::new(); let mut deadline_expired = false; - while !tasks.is_empty() { - match tokio::time::timeout_at(deadline, tasks.join_next_with_id()).await { - Ok(Some(Ok((task_id, result)))) => { - let Some(id) = task_owners.remove(&task_id) else { - failures.insert( - format!("task-{task_id}"), - "cleanup result had no tracked sandbox owner".to_string(), - ); - continue; - }; + while !tasks.is_empty() || pool_pending { + tokio::select! { + result = &mut pool_shutdown, if pool_pending => { + pool_pending = false; if let Err(error) = result { - failures.insert(id.to_string(), error.to_string()); + failures.insert("runtime-pool".to_string(), error.to_string()); } } - Ok(Some(Err(error))) => { - let key = task_owners - .remove(&error.id()) - .map_or_else(|| format!("task-{}", error.id()), |id| id.to_string()); - failures.insert(key, format!("cleanup task failed: {error}")); + task = tasks.join_next_with_id(), if !tasks.is_empty() => { + match task { + Some(Ok((task_id, result))) => { + let Some(id) = task_owners.remove(&task_id) else { + failures.insert( + format!("task-{task_id}"), + "cleanup result had no tracked sandbox owner".to_string(), + ); + continue; + }; + if let Err(error) = result { + failures.insert(id, error.to_string()); + } + } + Some(Err(error)) => { + let key = task_owners + .remove(&error.id()) + .unwrap_or_else(|| format!("task-{}", error.id())); + failures.insert(key, format!("cleanup task failed: {error}")); + } + None => {} + } } - Ok(None) => break, - Err(_) => { + _ = &mut deadline_sleep => { deadline_expired = true; break; } @@ -519,7 +551,7 @@ pub(crate) async fn shutdown_instances(state: &Arc, budget: Duratio } if deadline_expired { - let mut timed_out_owners = task_owners.values().copied().collect::>(); + let mut timed_out_owners = task_owners.values().cloned().collect::>(); tasks.abort_all(); while let Some(result) = tasks.join_next_with_id().await { match result { @@ -533,13 +565,14 @@ pub(crate) async fn shutdown_instances(state: &Arc, budget: Duratio }; timed_out_owners.remove(&id); if let Err(error) = result { - failures.insert(id.to_string(), error.to_string()); + failures.insert(id, error.to_string()); } } Err(error) => { let owner = task_owners.remove(&error.id()); - let key = - owner.map_or_else(|| format!("task-{}", error.id()), |id| id.to_string()); + let key = owner + .clone() + .unwrap_or_else(|| format!("task-{}", error.id())); if !error.is_cancelled() { if let Some(id) = owner { timed_out_owners.remove(&id); @@ -550,11 +583,18 @@ pub(crate) async fn shutdown_instances(state: &Arc, budget: Duratio } } for id in timed_out_owners { - failures.entry(id.to_string()).or_insert_with(|| { + failures.entry(id).or_insert_with(|| { format!("cleanup did not finish within the shared {budget:?} budget") }); } + if pool_pending { + pool_pending = false; + if let Err(error) = pool_shutdown.await { + failures.insert("runtime-pool".to_string(), error.to_string()); + } + } } + debug_assert!(!pool_pending); if failures.is_empty() { Ok(()) @@ -564,7 +604,7 @@ pub(crate) async fn shutdown_instances(state: &Arc, budget: Duratio .map(|(owner, error)| format!("{owner}: {error}")) .collect::>(); Err(BlazeDaemonError::RecoveryRequired(format!( - "daemon shutdown left {} sandbox cleanup operation(s) incomplete: {}", + "daemon shutdown left {} runtime cleanup operation(s) incomplete: {}", failures.len(), failures.join("; ") ))) @@ -839,8 +879,8 @@ mod tests { use std::collections::HashMap; use std::future; use std::path::{Path, PathBuf}; - use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; use std::time::Duration; use async_trait::async_trait; @@ -848,7 +888,7 @@ mod tests { use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; - use blaze_core::lifecycle::{BackendOwnership, OperationKind}; + use blaze_core::lifecycle::{BackendOwnership, OperationKind, RuntimeLocation}; use blaze_core::policy::{ BackendConfigs, FallbackOnMissingHook, PolicyEngine, PolicyFile, PolicyHooks, PolicyMatch, PolicyPool, PolicySelect, ResetMode, WorkloadClass, @@ -861,6 +901,7 @@ mod tests { use http_body_util::BodyExt; use crate::file_provider::FileStorageProvider; + use crate::runtime_pool::PoolPrototype; #[cfg(target_os = "linux")] use crate::spawner::BubblewrapSpawner; use crate::spawner::{ @@ -908,7 +949,7 @@ mod tests { min: 0, target: 0, max: 1, - warm_ttl: "30m".into(), + warm_ttl: Some("30m".into()), reset_mode: ResetMode::FullRecreate, }), checkpoint: None, @@ -978,6 +1019,111 @@ mod tests { .expect("created json") } + async fn wait_for_ready_runtime(state: &Arc) -> Uuid { + let runtime_root = state.state_dir.join("runtime-pool"); + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if state.manager.runtime_pool_status().ready > 0 + && let Ok(mut entries) = tokio::fs::read_dir(&runtime_root).await + { + while let Ok(Some(entry)) = entries.next_entry().await { + let Ok(instance_id) = Uuid::parse_str(&entry.file_name().to_string_lossy()) + else { + continue; + }; + let Ok(raw) = tokio::fs::read(entry.path().join("ownership.json")).await + else { + continue; + }; + let Ok(ownership) = serde_json::from_slice::(&raw) + else { + continue; + }; + if ownership["phase"]["kind"] == "ready" { + return instance_id; + } + } + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + }) + .await + .expect("runtime pool produces a ready slot") + } + + fn warm_runtime_state( + temp: &tempfile::TempDir, + prefork: bool, + spawner: DynSpawner, + ) -> (Arc, PathBuf) { + let mut config = test_config(temp); + config.storage.pool_size = 1; + config.storage.prefork = prefork; + std::fs::create_dir_all(config.daemon.state_dir.join("runtime-pool")) + .expect("runtime pool root"); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir.clone(), + )); + ( + build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, spawner), + BackendKind::Mock, + storage, + ), + instances_dir, + ) + } + + async fn assert_warm_runtime_round_trip(prefork: bool) { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state(&temp, prefork, Arc::new(MockSpawner)); + + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let claimed = created_json(&state, &test_request()).await; + let claimed_id = + Uuid::parse_str(claimed["instance"]["id"].as_str().expect("claimed ID")).expect("UUID"); + + assert_eq!(claimed_id, ready_id); + assert_eq!(claimed["start_path"], "warm"); + assert_eq!(claimed["instance"]["runtime_location"], "warm-pool"); + assert!(claimed["instance"]["runtime_owner_token"].is_string()); + + state.manager.begin_shutdown(); + assert!( + state + .manager + .destroy(claimed_id) + .await + .expect("destroy claim") + ); + let terminal = state.manager.get(claimed_id).expect("terminal lifecycle"); + assert!(terminal.is_clean_terminal()); + assert!(!instances_dir.join(claimed_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(claimed_id.to_string()) + .exists() + ); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(".cleanup") + .join(claimed_id.to_string()) + .exists() + ); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown remaining owners"); + } + async fn dispatched_json( state: &Arc, method: Method, @@ -1489,6 +1635,97 @@ mod tests { release_count: Arc, } + struct PoolWorkerReleaseStorage { + inner: FileStorageProvider, + acquire_count: AtomicUsize, + residual_attempt: usize, + delayed_id: Mutex>, + release_started: Arc, + release_active: Arc, + release_completed: Arc, + release_delay: Duration, + } + + struct ActiveRelease { + active: Arc, + } + + impl Drop for ActiveRelease { + fn drop(&mut self) { + self.active.fetch_sub(1, Ordering::AcqRel); + } + } + + #[async_trait] + impl StorageProvider for PoolWorkerReleaseStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + let attempt = self.acquire_count.fetch_add(1, Ordering::AcqRel) + 1; + let slot = self.inner.acquire(opts).await?; + if attempt == self.residual_attempt { + *self.delayed_id.lock().expect("delayed ID") = Some(slot.id.clone()); + return Err(StorageAcquireError::with_residual( + BlazeError::StorageError { + msg: "injected pool build residual".to_string(), + }, + slot, + )); + } + Ok(slot) + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + let delayed = + self.delayed_id.lock().expect("delayed ID").as_deref() == Some(instance_id); + if delayed { + self.release_active.fetch_add(1, Ordering::AcqRel); + let _active = ActiveRelease { + active: self.release_active.clone(), + }; + self.release_started.fetch_add(1, Ordering::AcqRel); + tokio::time::sleep(self.release_delay).await; + self.inner.release_by_id(instance_id).await?; + self.release_completed.fetch_add(1, Ordering::AcqRel); + return Ok(()); + } + self.inner.release_by_id(instance_id).await + } + + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> blaze_core::Result> { + self.inner.list_owned_ids().await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + #[async_trait] impl StorageProvider for CountingStorage { async fn probe(&self) -> blaze_core::Result { @@ -1702,6 +1939,198 @@ mod tests { } } + #[tokio::test] + async fn non_prefork_runtime_claim_completes_create_and_destroy() { + assert_warm_runtime_round_trip(false).await; + } + + #[tokio::test] + async fn prefork_runtime_claim_completes_create_and_destroy() { + assert_warm_runtime_round_trip(true).await; + } + + #[tokio::test] + async fn omitted_policy_ttl_is_resolved_in_create_response() { + let temp = tempfile::tempdir().expect("temp"); + let mut config = test_config(&temp); + config.pool.default_warm_ttl = "1h".into(); + let instances_dir = config.storage.instances_dir.clone(); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + instances_dir, + )); + let mut policy = test_policy(BackendKind::Mock, true); + policy.pool.as_mut().expect("pool").warm_ttl = None; + let state = build_test_state( + config, + policy, + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let created = created_json(&state, &test_request()).await; + assert_eq!(created["decision"]["pool"]["warm_ttl"], "1h"); + let id = Uuid::parse_str(created["instance"]["id"].as_str().expect("instance ID")) + .expect("UUID"); + assert!(state.manager.destroy(id).await.expect("destroy instance")); + shutdown_instances(&state, Duration::from_millis(100)) + .await + .expect("no owners remain"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn visible_lifecycle_publish_error_keeps_one_cleanup_owner() { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state(&temp, false, Arc::new(MockSpawner)); + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let hook = crate::failpoint::TestFailpoint::new(&["warm-runtime-lifecycle-publish-result"]); + + let error = hook + .run(create_instance(&state, &test_request())) + .await + .expect_err("visible lifecycle publication must be compensated"); + + assert!(error.to_string().contains("publication reported an error")); + let terminal = state.manager.get(ready_id).expect("terminal lifecycle"); + assert!(terminal.is_clean_terminal()); + assert!(state.manager.backend_owner(ready_id).is_none()); + assert!(!instances_dir.join(ready_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .exists() + ); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(".cleanup") + .join(ready_id.to_string()) + .exists() + ); + + state.manager.begin_shutdown(); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown remaining owners"); + } + + #[cfg(unix)] + #[tokio::test] + async fn ambiguous_lifecycle_publish_remains_counted_until_restart() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = warm_runtime_state( + &temp, + false, + Arc::new(CountingSpawner { + kill_count: Arc::new(AtomicUsize::new(0)), + orphan_cleanup_count: Arc::new(AtomicUsize::new(0)), + }), + ); + let _bootstrap = created_json(&state, &test_request()).await; + let ready_id = wait_for_ready_runtime(&state).await; + let external = tempfile::tempdir().expect("external lifecycle owner"); + symlink(external.path(), state.state_dir.join(ready_id.to_string())) + .expect("linked lifecycle owner"); + + let error = create_instance(&state, &test_request()) + .await + .expect_err("ambiguous lifecycle publication must stop the claim"); + + assert!(error.to_string().contains("publication was ambiguous")); + let status = state.manager.runtime_pool_status(); + assert_eq!(status.ready, 0); + assert_eq!(status.leased, 0); + assert_eq!(status.unresolved, 1); + assert_eq!(status.deficit, 0); + tokio::time::sleep(Duration::from_millis(20)).await; + assert_eq!(state.manager.runtime_pool_status().unresolved, 1); + assert!(instances_dir.join(ready_id.to_string()).exists()); + + state.manager.begin_shutdown(); + let shutdown = shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect_err("shutdown must report the unresolved owner"); + assert!(shutdown.to_string().contains(&ready_id.to_string())); + assert!( + shutdown + .to_string() + .contains("unresolved lifecycle publication") + ); + assert_eq!(state.manager.runtime_pool_status().unresolved, 1); + } + + #[tokio::test] + async fn failed_runtime_claim_retains_one_recoverable_owner() { + let temp = tempfile::tempdir().expect("temp"); + let (state, instances_dir) = + warm_runtime_state(&temp, false, Arc::new(PartialSpawnSpawner)); + + let _bootstrap_error = create_instance(&state, &test_request()) + .await + .expect_err("bootstrap spawn fails after pool configuration"); + let ready_id = wait_for_ready_runtime(&state).await; + let error = create_instance(&state, &test_request()) + .await + .expect_err("runtime claim spawn fails"); + + assert!(error.to_string().contains("cleanup incomplete")); + let retained = state.manager.get(ready_id).expect("retained lifecycle"); + assert_eq!(retained.runtime_location, RuntimeLocation::WarmPool); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Destroy) + ); + assert!(!retained.is_clean_terminal()); + let ownership: serde_json::Value = serde_json::from_slice( + &std::fs::read( + state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .join("ownership.json"), + ) + .expect("retained ownership"), + ) + .expect("ownership JSON"); + assert_eq!(ownership["phase"]["kind"], "lifecycle-cleanup"); + + assert!( + state + .manager + .destroy(ready_id) + .await + .expect("retry retained cleanup") + ); + assert!( + state + .manager + .get(ready_id) + .expect("terminal lifecycle") + .is_clean_terminal() + ); + assert!(!instances_dir.join(ready_id.to_string()).exists()); + assert!( + !state + .state_dir + .join("runtime-pool") + .join(ready_id.to_string()) + .exists() + ); + + state.manager.begin_shutdown(); + shutdown_instances(&state, Duration::from_secs(1)) + .await + .expect("shutdown bootstrap owner"); + } + /// When multiple backend binaries exist on disk but the daemon probed /// Firecracker at boot, only Firecracker should be reported available /// and selected — even if policy prioritizes bubblewrap higher. @@ -2101,7 +2530,7 @@ mod tests { min: 0, target: 0, max: 1, - warm_ttl: "30m".into(), + warm_ttl: Some("30m".into()), reset_mode: ResetMode::FullRecreate, }), checkpoint: None, @@ -2613,6 +3042,86 @@ mod tests { .expect("no ownership remains"); } + #[tokio::test] + async fn daemon_shutdown_joins_an_active_pool_worker_before_returning() { + let temp = tempfile::tempdir().expect("temp"); + let mut config = test_config(&temp); + config.storage.pool_size = 1; + config.storage.prefork = false; + std::fs::create_dir_all(config.daemon.state_dir.join("runtime-pool")) + .expect("runtime pool root"); + let release_started = Arc::new(AtomicUsize::new(0)); + let release_active = Arc::new(AtomicUsize::new(0)); + let release_completed = Arc::new(AtomicUsize::new(0)); + let storage = Arc::new(PoolWorkerReleaseStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + acquire_count: AtomicUsize::new(0), + residual_attempt: 1, + delayed_id: Mutex::new(None), + release_started: release_started.clone(), + release_active: release_active.clone(), + release_completed: release_completed.clone(), + release_delay: Duration::from_secs(1), + }); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, true), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + state + .manager + .configure_runtime_pool_for_test(PoolPrototype { + image_digest: "sha256:pool-shutdown".to_string(), + policy_name: "pool-shutdown".to_string(), + workload_class: WorkloadClass::AgentTool, + templates: Vec::new(), + kernel_hooks: Vec::new(), + binary_path: PathBuf::from("/unused"), + runtime_backend: BackendKind::Mock, + backend: BackendConfigs::default(), + vm: None, + warm_ttl: Duration::from_secs(60), + }) + .expect("configure runtime pool"); + tokio::time::timeout(Duration::from_secs(2), async { + while release_active.load(Ordering::Acquire) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("pool worker starts delayed cleanup"); + + let error = shutdown_instances(&state, Duration::from_millis(40)) + .await + .expect_err("pool cleanup exceeds the shared budget"); + + assert!(error.to_string().contains("runtime-pool")); + assert_eq!(release_started.load(Ordering::Acquire), 1); + assert_eq!(release_completed.load(Ordering::Acquire), 0); + assert_eq!( + release_active.load(Ordering::Acquire), + 0, + "shutdown returned while the pool worker was still running" + ); + assert!(!state.manager.runtime_pool_has_tracked_worker()); + let status = state.manager.runtime_pool_status(); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.quarantined, 1); + + state + .manager + .shutdown_runtime_pool_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry releases the retained pool owner"); + assert_eq!(release_completed.load(Ordering::Acquire), 1); + assert_eq!(state.manager.runtime_pool_status().quarantined, 0); + } + #[tokio::test(start_paused = true)] async fn daemon_shutdown_joins_all_owners_within_one_budget() { let temp = tempfile::tempdir().expect("temp"); diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index f0050fa789..1119b3a8fe 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use std::time::Duration; use blaze_core::backend::BackendKind; -use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode}; +use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode, validate_runtime_storage_paths}; use blaze_core::kernel::HookRegistry; use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; @@ -191,6 +191,13 @@ pub async fn run(config_path: &Path) -> Result<()> { active_backend, storage, )?); + let reconciled_slots = state.manager.reconcile_runtime_pool_startup().await?; + if reconciled_slots > 0 { + tracing::warn!( + runtimes = reconciled_slots, + "reconciled unclaimed runtime slots" + ); + } let reconciliation = state.manager.reconcile_startup().await; tracing::info!( attempted = reconciliation.attempted, @@ -229,9 +236,14 @@ fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { std::fs::create_dir_all(&cfg.template.dir)?; std::fs::create_dir_all(&cfg.storage.images_dir)?; std::fs::create_dir_all(&cfg.storage.instances_dir)?; + let state_dir = std::fs::canonicalize(&cfg.daemon.state_dir)?; let images_dir = std::fs::canonicalize(&cfg.storage.images_dir)?; let instances_dir = std::fs::canonicalize(&cfg.storage.instances_dir)?; - blaze_core::config::validate_storage_paths(&images_dir, &instances_dir)?; + let runtime_root = state_dir.join("runtime-pool"); + validate_runtime_storage_paths(&runtime_root, &images_dir, &instances_dir)?; + std::fs::create_dir_all(&runtime_root)?; + let runtime_root = std::fs::canonicalize(runtime_root)?; + validate_runtime_storage_paths(&runtime_root, &images_dir, &instances_dir)?; if let Some(parent) = cfg.daemon.socket.parent() { std::fs::create_dir_all(parent)?; } @@ -482,6 +494,7 @@ fn reload_policies(state: &Arc) -> Result<()> { #[cfg(test)] mod tests { use std::future; + use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -497,6 +510,53 @@ mod tests { } } + fn directory_config(root: &Path) -> DaemonConfig { + let mut config = DaemonConfig::default(); + config.daemon.state_dir = root.join("state"); + config.daemon.socket = root.join("run").join("blazed.sock"); + config.storage.images_dir = root.join("images"); + config.storage.instances_dir = root.join("instances"); + config.template.dir = root.join("templates"); + config.policy.dir = root.join("policies"); + config + } + + #[test] + fn ensure_dirs_accepts_disjoint_runtime_and_storage_roots() { + let temp = tempfile::tempdir().expect("tempdir"); + let config = directory_config(temp.path()); + + ensure_dirs(&config).expect("disjoint ownership roots"); + + assert!(config.daemon.state_dir.join("runtime-pool").is_dir()); + assert!(config.storage.images_dir.is_dir()); + assert!(config.storage.instances_dir.is_dir()); + } + + #[cfg(unix)] + #[test] + fn ensure_dirs_rejects_canonical_runtime_storage_alias() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let config = directory_config(temp.path()); + std::fs::create_dir_all(&config.daemon.state_dir).expect("state dir"); + std::fs::create_dir_all(&config.storage.instances_dir).expect("instances dir"); + symlink( + &config.storage.instances_dir, + config.daemon.state_dir.join("runtime-pool"), + ) + .expect("runtime alias"); + + let error = ensure_dirs(&config).expect_err("canonical overlap must be rejected"); + + assert!(matches!( + error, + BlazeDaemonError::Core(blaze_core::BlazeError::ConfigError { .. }) + )); + assert!(error.to_string().contains("must be disjoint")); + } + #[tokio::test] async fn shutdown_waits_for_inflight_connection_task() { let mut connections = ConnectionSupervisor::new(); diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index f333027bba..7add3765b0 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -18,6 +18,7 @@ mod file_provider; mod guest; mod metrics; mod request_body; +mod runtime_pool; mod sandbox; mod spawner; mod state; diff --git a/src/blaze/crates/blazed/src/runtime_pool.rs b/src/blaze/crates/blazed/src/runtime_pool.rs new file mode 100644 index 0000000000..77aeee3bc4 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Runtime slot ownership, recovery, and capacity management. + +mod pool; +mod recovery; + +#[cfg(test)] +pub(crate) use pool::RuntimePoolStatus; +pub(crate) use pool::{PoolPrototype, RuntimePoolLease, RuntimeWarmPool}; +pub(crate) use recovery::{ + DurableRuntimeOwner, begin_lifecycle_cleanup, reconcile_runtime_slots, + remove_lifecycle_tombstone, runtime_dir, tombstone_lifecycle_slot, +}; diff --git a/src/blaze/crates/blazed/src/runtime_pool/pool.rs b/src/blaze/crates/blazed/src/runtime_pool/pool.rs new file mode 100644 index 0000000000..09553357e6 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool/pool.rs @@ -0,0 +1,2451 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Bounded background construction and ownership transfer for warm runtimes. + +use std::collections::{BTreeSet, VecDeque}; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::lifecycle::BackendOwnership; +use blaze_core::policy::{BackendConfigs, VmConfig, WorkloadClass}; +use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; +use blaze_core::{BlazeError, Result}; +use tokio::sync::{Mutex as AsyncMutex, Notify}; +use tokio::task::JoinHandle; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use super::recovery::{ + RuntimeSlotOwnership, RuntimeSlotPhase, finish_ownership_handoff, read_ownership, + remove_pool_tombstone, tombstone_pool_slot, write_ownership, +}; +use crate::guest::{GuestClient, MAX_GUEST_FILE_BYTES}; +use crate::spawner::{DynBackendInstance, SpawnerRegistry}; + +const RETRY_BASE_DELAY: Duration = Duration::from_millis(100); +const RETRY_MAX_DELAY: Duration = Duration::from_secs(30); +const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// Immutable inputs shared by every slot in one runtime pool generation. +#[derive(Debug, Clone)] +pub(crate) struct PoolPrototype { + pub(crate) image_digest: String, + pub(crate) policy_name: String, + pub(crate) workload_class: WorkloadClass, + pub(crate) templates: Vec, + pub(crate) kernel_hooks: Vec, + pub(crate) binary_path: PathBuf, + pub(crate) runtime_backend: BackendKind, + pub(crate) backend: BackendConfigs, + pub(crate) vm: Option, + pub(crate) warm_ttl: Duration, +} + +impl PoolPrototype { + fn fingerprint(&self) -> Result> { + serde_json::to_vec(&( + &self.image_digest, + &self.policy_name, + self.workload_class, + &self.templates, + &self.kernel_hooks, + &self.binary_path, + self.runtime_backend, + &self.backend, + &self.vm, + self.warm_ttl.as_nanos(), + )) + .map_err(|error| pool_error(format!("serialize runtime pool prototype: {error}"))) + } +} + +/// Resources transferred from the pool into lifecycle ownership. +pub(crate) struct RuntimePoolSlot { + pub(crate) instance_id: Uuid, + pub(crate) storage: StorageSlot, + pub(crate) backend: Option, + pub(crate) run_dir: PathBuf, + pub(crate) runtime_backend: BackendKind, + pub(crate) backend_ownership: BackendOwnership, + ready_at: Instant, +} + +struct UnresolvedHandoff { + slot: RuntimePoolSlot, + token: Uuid, + reason: String, +} + +struct CleanupSlot { + ownership: RuntimeSlotOwnership, + backend: Option, + run_dir: PathBuf, + authority: PoolCleanupAuthority, +} + +impl CleanupSlot { + fn from_runtime(slot: RuntimePoolSlot) -> Self { + Self::from_runtime_with_authority(slot, PoolCleanupAuthority::Ready) + } + + fn from_runtime_with_authority(slot: RuntimePoolSlot, authority: PoolCleanupAuthority) -> Self { + let mut ownership = RuntimeSlotOwnership::new(slot.instance_id, slot.runtime_backend); + ownership.backend_ownership = slot.backend_ownership; + ownership.storage_owned = true; + ownership.phase = RuntimeSlotPhase::Ready; + Self { + ownership, + backend: slot.backend, + run_dir: slot.run_dir, + authority, + } + } +} + +#[derive(Clone, Copy)] +enum PoolCleanupAuthority { + Build, + Ready, + Handoff(Uuid), +} + +#[derive(Default)] +struct PoolState { + ready: VecDeque, + quarantined: VecDeque, + unresolved: VecDeque, + building: BTreeSet, + leased: BTreeSet, + cleanup_pending: usize, + prototype: Option, + prototype_fingerprint: Option>, + generation: u64, + consecutive_build_failures: u32, + consecutive_cleanup_failures: u32, + shutting_down: bool, +} + +impl PoolState { + fn physical_count(&self) -> usize { + self.ready + .len() + .saturating_add(self.building.len()) + .saturating_add(self.leased.len()) + .saturating_add(self.quarantined.len()) + .saturating_add(self.unresolved.len()) + .saturating_add(self.cleanup_pending) + } +} + +/// Bounded worker that retains every incomplete resource for retry. +pub(crate) struct RuntimeWarmPool { + target: usize, + prefork: bool, + rootfs_size: u64, + mem_size: u64, + runtime_root: PathBuf, + storage: Arc, + spawners: Arc, + default_warm_ttl: Duration, + gc_interval: Duration, + state: Mutex, + maintenance: AsyncMutex<()>, + shutdown: AsyncMutex<()>, + cancellation: CancellationToken, + wake: Notify, + worker: Mutex>>, +} + +struct WorkerJoinGuard<'a> { + worker_slot: &'a Mutex>>, + handle: Option>, +} + +impl<'a> WorkerJoinGuard<'a> { + fn new(worker_slot: &'a Mutex>>, handle: Option>) -> Self { + Self { + worker_slot, + handle, + } + } + + fn disarm(&mut self) { + self.handle.take(); + } +} + +impl Drop for WorkerJoinGuard<'_> { + fn drop(&mut self) { + let Some(handle) = self.handle.take() else { + return; + }; + handle.abort(); + let mut worker = match self.worker_slot.lock() { + Ok(worker) => worker, + Err(poisoned) => poisoned.into_inner(), + }; + if worker.is_none() { + *worker = Some(handle); + } else { + tracing::error!( + "runtime pool worker slot was occupied while retaining an aborted worker" + ); + } + } +} + +impl RuntimeWarmPool { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + target: usize, + prefork: bool, + rootfs_size: u64, + mem_size: u64, + runtime_root: PathBuf, + storage: Arc, + spawners: Arc, + default_warm_ttl: Duration, + gc_interval: Duration, + cancellation: CancellationToken, + ) -> Result> { + if target > 0 && !storage.supports_runtime_pool_recovery() { + return Err(pool_error( + "configured storage provider does not expose runtime slot cleanup inventory", + )); + } + Ok(Arc::new(Self { + target, + prefork, + rootfs_size, + mem_size, + runtime_root, + storage, + spawners, + default_warm_ttl, + gc_interval, + state: Mutex::new(PoolState::default()), + maintenance: AsyncMutex::new(()), + shutdown: AsyncMutex::new(()), + cancellation, + wake: Notify::new(), + worker: Mutex::new(None), + })) + } + + /// Fix the first compatible build shape and start maintenance on demand. + pub(crate) fn configure(self: &Arc, prototype: PoolPrototype) -> Result { + if self.target == 0 { + return Ok(false); + } + let fingerprint = prototype.fingerprint()?; + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down { + return Err(pool_error("runtime pool is shutting down")); + } + match state.prototype_fingerprint.as_deref() { + None => { + state.prototype = Some(prototype); + state.prototype_fingerprint = Some(fingerprint); + } + Some(existing) if existing == fingerprint.as_slice() => {} + Some(_) => return Ok(false), + } + drop(state); + self.ensure_worker()?; + self.wake.notify_one(); + Ok(true) + } + + /// Lease one ready slot while a synchronous guard protects cancellation. + pub(crate) async fn acquire(self: &Arc) -> Result> { + loop { + let (slot, warm_ttl) = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + let warm_ttl = state + .prototype + .as_ref() + .map(|prototype| prototype.warm_ttl) + .unwrap_or(self.default_warm_ttl); + let slot = state.ready.pop_front(); + if let Some(slot) = &slot { + state.leased.insert(slot.instance_id); + } + (slot, warm_ttl) + }; + self.wake.notify_one(); + let Some(slot) = slot else { + return Ok(None); + }; + let lease = RuntimePoolLease::new(self.clone(), slot); + if lease.slot()?.ready_at.elapsed() >= warm_ttl { + tracing::info!( + instance = %lease.slot()?.instance_id, + "discarding expired runtime slot before claim" + ); + lease.quarantine(); + continue; + } + let Some(backend) = lease.slot()?.backend.as_ref().cloned() else { + return Ok(Some(lease)); + }; + let live = tokio::time::timeout(GUEST_REQUEST_TIMEOUT, backend.try_wait()).await; + match live { + Ok(Ok(None)) => return Ok(Some(lease)), + Ok(Ok(Some(result))) => tracing::warn!( + instance = %lease.slot()?.instance_id, + exit_code = ?result.exit_code, + signal = ?result.signal, + "discarding exited prefork runtime" + ), + Ok(Err(error)) => tracing::warn!( + instance = %lease.slot()?.instance_id, + %error, + "discarding runtime after liveness check failed" + ), + Err(_) => tracing::warn!( + instance = %lease.slot()?.instance_id, + "discarding runtime after liveness check timed out" + ), + } + lease.quarantine(); + } + } + + pub(crate) fn begin_shutdown(&self) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.shutting_down = true; + state.generation = state.generation.wrapping_add(1); + drop(state); + self.cancellation.cancel(); + self.wake.notify_waiters(); + } + + pub(crate) fn default_warm_ttl(&self) -> Duration { + self.default_warm_ttl + } + + /// Stop maintenance and release pool-owned slots before one deadline. + pub(crate) async fn shutdown_until( + self: &Arc, + deadline: tokio::time::Instant, + ) -> Result<()> { + self.begin_shutdown(); + let _shutdown = tokio::time::timeout_at(deadline, self.shutdown.lock()) + .await + .map_err(|_| { + pool_error("runtime pool shutdown coordination exceeded the shared deadline") + })?; + let worker = self + .worker + .lock() + .map_err(|_| pool_error("runtime pool worker lock poisoned"))? + .take(); + let mut worker = WorkerJoinGuard::new(&self.worker, worker); + let mut errors = Vec::new(); + if let Some(handle) = worker.handle.as_mut() { + match tokio::time::timeout_at(deadline, &mut *handle).await { + Ok(Ok(())) => { + worker.disarm(); + } + Ok(Err(error)) if error.is_cancelled() => { + worker.disarm(); + } + Ok(Err(error)) => { + worker.disarm(); + errors.push(format!( + "runtime pool worker failed while stopping: {error}" + )); + } + Err(_) => { + handle.abort(); + let result = (&mut *handle).await; + worker.disarm(); + if let Err(error) = result + && !error.is_cancelled() + { + errors.push(format!( + "runtime pool worker failed while stopping: {error}" + )); + } + errors.push( + "runtime pool worker exceeded the shared shutdown deadline".to_string(), + ); + } + } + } + + loop { + let (building, leased) = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + (state.building.len(), state.leased.len()) + }; + if building == 0 && leased == 0 { + break; + } + if tokio::time::timeout_at(deadline, self.wake.notified()) + .await + .is_err() + { + if building != 0 { + errors.push(format!( + "{building} runtime pool build(s) exceeded the shared shutdown deadline" + )); + } + if leased != 0 { + errors.push(format!( + "{leased} runtime pool lease(s) exceeded the shared shutdown deadline" + )); + } + break; + } + } + + let unresolved = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state + .unresolved + .iter() + .map(|handoff| { + format!( + "{} (token {}, {})", + handoff.slot.instance_id, handoff.token, handoff.reason + ) + }) + .collect::>() + }; + if !unresolved.is_empty() { + errors.push(format!( + "{} runtime pool owner(s) have unresolved lifecycle publication: {}", + unresolved.len(), + unresolved.join(", ") + )); + } + + let maintenance = match tokio::time::timeout_at(deadline, self.maintenance.lock()).await { + Ok(maintenance) => Some(maintenance), + Err(_) => { + errors.push( + "runtime pool maintenance lock exceeded the shared shutdown deadline" + .to_string(), + ); + None + } + }; + if let Some(_maintenance) = maintenance { + let attempts = { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.ready.len().saturating_add(state.quarantined.len()) + }; + for _ in 0..attempts { + let cleanup = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + let cleanup = state + .quarantined + .pop_front() + .or_else(|| state.ready.pop_front().map(CleanupSlot::from_runtime)); + if cleanup.is_some() { + state.cleanup_pending = state.cleanup_pending.saturating_add(1); + } + cleanup + }; + let Some(cleanup) = cleanup else { + break; + }; + let instance_id = cleanup.ownership.instance_id; + let mut cleanup = CleanupGuard::new(self.clone(), cleanup); + match tokio::time::timeout_at(deadline, self.cleanup_slot(cleanup.slot_mut()?)) + .await + { + Ok(Ok(())) => {} + Ok(Err(error)) => { + errors.push(format!("{instance_id}: {error}")); + cleanup.retry(); + continue; + } + Err(_) => { + errors.push(format!( + "{instance_id}: cleanup exceeded the shared shutdown deadline" + )); + cleanup.retry(); + break; + } + } + cleanup.complete(); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(pool_error(errors.join("; "))) + } + } + + #[cfg(test)] + pub(crate) fn status(&self) -> RuntimePoolStatus { + match self.state.lock() { + Ok(state) => RuntimePoolStatus::from_state(self.target, &state), + Err(poisoned) => RuntimePoolStatus::from_state(self.target, &poisoned.into_inner()), + } + } + + #[cfg(test)] + pub(crate) fn has_tracked_worker(&self) -> bool { + match self.worker.lock() { + Ok(worker) => worker.is_some(), + Err(poisoned) => poisoned.into_inner().is_some(), + } + } + + fn ensure_worker(self: &Arc) -> Result<()> { + let state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down { + return Err(pool_error("runtime pool is shutting down")); + } + let mut worker = self + .worker + .lock() + .map_err(|_| pool_error("runtime pool worker lock poisoned"))?; + if worker.as_ref().is_some_and(JoinHandle::is_finished) { + worker.take(); + } + if worker.is_none() { + let pool = self.clone(); + *worker = Some(tokio::spawn(async move { + pool.run_worker().await; + })); + } + Ok(()) + } + + async fn run_worker(self: Arc) { + let mut gc = tokio::time::interval(self.gc_interval); + gc.set_missed_tick_behavior(MissedTickBehavior::Skip); + gc.tick().await; + loop { + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = self.wake.notified() => {} + _ = gc.tick() => {} + } + loop { + if self.cancellation.is_cancelled() { + return; + } + match self.maintain_once().await { + Ok(Maintenance::Progress) => continue, + Ok(Maintenance::Idle) => break, + Ok(Maintenance::RetryAfter(delay)) => { + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = tokio::time::sleep(delay) => {} + } + } + Err(error) => { + tracing::warn!(%error, "runtime pool maintenance failed"); + tokio::select! { + _ = self.cancellation.cancelled() => return, + _ = tokio::time::sleep(RETRY_BASE_DELAY) => {} + } + } + } + } + } + } + + async fn maintain_once(self: &Arc) -> Result { + let _maintenance = self.maintenance.lock().await; + let action = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + if state.shutting_down || state.prototype.is_none() { + return Ok(Maintenance::Idle); + } + let warm_ttl = state + .prototype + .as_ref() + .map(|prototype| prototype.warm_ttl) + .unwrap_or(self.default_warm_ttl); + let now = Instant::now(); + let mut retained = VecDeque::new(); + while let Some(slot) = state.ready.pop_front() { + if now.duration_since(slot.ready_at) >= warm_ttl { + state.quarantined.push_back(CleanupSlot::from_runtime(slot)); + } else { + retained.push_back(slot); + } + } + state.ready = retained; + if let Some(cleanup) = state.quarantined.pop_front() { + state.cleanup_pending = state.cleanup_pending.saturating_add(1); + PoolAction::Cleanup(cleanup) + } else if state.physical_count() < self.target { + PoolAction::Build(state.generation) + } else { + PoolAction::Idle + } + }; + + match action { + PoolAction::Idle => Ok(Maintenance::Idle), + PoolAction::Build(generation) => match self.build_slot(generation).await { + Ok(()) => { + if let Ok(mut state) = self.state.lock() { + state.consecutive_build_failures = 0; + } + Ok(Maintenance::Progress) + } + Err(error) => { + let delay = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.consecutive_build_failures = + state.consecutive_build_failures.saturating_add(1); + build_retry_delay(state.consecutive_build_failures) + }; + tracing::warn!( + %error, + retry_delay_ms = delay.as_millis(), + "runtime slot build failed" + ); + Ok(Maintenance::RetryAfter(delay)) + } + }, + PoolAction::Cleanup(cleanup) => { + let mut cleanup = CleanupGuard::new(self.clone(), cleanup); + match self.cleanup_slot(cleanup.slot_mut()?).await { + Ok(()) => { + cleanup.complete(); + Ok(Maintenance::Progress) + } + Err(error) => { + let delay = { + let mut state = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.consecutive_cleanup_failures = + state.consecutive_cleanup_failures.saturating_add(1); + build_retry_delay(state.consecutive_cleanup_failures) + }; + cleanup.retry(); + tracing::warn!( + %error, + retry_delay_ms = delay.as_millis(), + "runtime slot cleanup will be retried" + ); + Ok(Maintenance::RetryAfter(delay)) + } + } + } + } + } + + async fn build_slot(self: &Arc, generation: u64) -> Result<()> { + crate::failpoint::storage("pool-build")?; + let prototype = self + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))? + .prototype + .clone() + .ok_or_else(|| pool_error("runtime pool has no build prototype"))?; + let instance_id = Uuid::new_v4(); + let run_dir = self.runtime_root.join(instance_id.to_string()); + std::fs::create_dir(&run_dir)?; + let ownership = RuntimeSlotOwnership::new(instance_id, prototype.runtime_backend); + let mut build = BuildGuard::new(self.clone(), run_dir, ownership); + std::fs::File::open(&self.runtime_root)?.sync_all()?; + build.persist().await?; + + // Acquire may be cancelled after the provider has created artifacts + // but before it can return a residual slot. Claim cleanup authority + // first; recovery-capable providers make release-by-ID safe when no + // artifact was created. + build.ownership.storage_owned = true; + build.persist().await?; + let acquire_opts = AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: self.rootfs_size, + mem_size: self.mem_size, + }; + let acquire = self.storage.acquire(&acquire_opts); + let storage = match tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error("runtime slot build cancelled during storage acquire")); + } + result = acquire => result, + } { + Ok(storage) => storage, + Err(error) => { + let (source, residual) = error.into_parts(); + if let Some(residual) = residual { + build.storage = Some(residual); + if let Err(persist) = build.persist().await { + return Err(pool_error(format!( + "{source}; retain residual storage journal: {persist}" + ))); + } + } + return Err(source); + } + }; + build.storage = Some(storage); + build.persist().await?; + + if self.prefork { + let spawner = self + .spawners + .get(prototype.runtime_backend) + .ok_or_else(|| { + pool_error(format!( + "no spawner registered for runtime backend {}", + prototype.runtime_backend + )) + })?; + build.ownership.backend_ownership = BackendOwnership::Starting; + build.persist().await?; + tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error( + "runtime slot build cancelled during backend preparation" + )); + } + result = spawner.prepare_spawn(&build.run_dir) => result?, + } + let spawn_request = SpawnRequest { + instance_id, + run_dir: build.run_dir.clone(), + binary_path: prototype.binary_path, + storage: build + .storage + .as_ref() + .ok_or_else(|| pool_error("runtime slot lost storage ownership"))? + .clone(), + backend: prototype.backend, + vm: prototype.vm, + }; + let spawn = tokio::select! { + _ = self.cancellation.cancelled() => { + return Err(pool_error( + "runtime slot build cancelled during backend spawn" + )); + } + result = spawner.spawn(spawn_request) => result, + }; + let backend = match spawn { + Ok(backend) => backend, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + build.backend = Some(owner); + build.ownership.backend_ownership = BackendOwnership::Running; + } else { + build.ownership.backend_ownership = BackendOwnership::Stopped; + } + if let Err(persist) = build.persist().await { + return Err(pool_error(format!( + "{source}; retain failed backend journal: {persist}" + ))); + } + return Err(source); + } + }; + build.ownership.backend_ownership = BackendOwnership::Running; + build.backend = Some(backend.clone()); + build.persist().await?; + if backend.backend() != prototype.runtime_backend { + return Err(pool_error(format!( + "runtime slot requested {} but spawner returned {}", + prototype.runtime_backend, + backend.backend() + ))); + } + let socket = backend.guest_socket_path(); + if !socket.as_os_str().is_empty() { + GuestClient::new( + socket.to_path_buf(), + GUEST_REQUEST_TIMEOUT, + MAX_GUEST_FILE_BYTES, + ) + .wait_ready(GUEST_REQUEST_TIMEOUT, &self.cancellation) + .await + .map_err(|error| pool_error(format!("prefork guest readiness failed: {error}")))?; + } + } + + build.ownership.phase = RuntimeSlotPhase::Ready; + build.persist().await?; + build.publish(generation) + } + + async fn cleanup_slot(&self, slot: &mut CleanupSlot) -> Result<()> { + let persisted = match read_ownership(&slot.run_dir, slot.ownership.instance_id).await { + Ok(persisted) => Some(persisted), + Err(error) => { + let ownership_path = slot.run_dir.join("ownership.json"); + let journal_missing = matches!( + tokio::fs::symlink_metadata(&ownership_path).await, + Err(io_error) if io_error.kind() == std::io::ErrorKind::NotFound + ); + if matches!(slot.authority, PoolCleanupAuthority::Build) && journal_missing { + None + } else { + return Err(pool_error(error)); + } + } + }; + let cleanup_committed = persisted + .as_ref() + .is_some_and(|ownership| ownership.phase == RuntimeSlotPhase::PoolCleanup); + if let Some(persisted) = persisted { + if persisted.backend != slot.ownership.backend { + return Err(pool_error(format!( + "runtime slot {} changed backend from {} to {} before pool cleanup", + slot.ownership.instance_id, slot.ownership.backend, persisted.backend + ))); + } + let phase_allowed = match (slot.authority, persisted.phase) { + (PoolCleanupAuthority::Build, RuntimeSlotPhase::Building) + | (PoolCleanupAuthority::Build, RuntimeSlotPhase::Ready) + | (PoolCleanupAuthority::Ready, RuntimeSlotPhase::Ready) + | (_, RuntimeSlotPhase::PoolCleanup) => true, + (PoolCleanupAuthority::Handoff(expected), RuntimeSlotPhase::Handoff { token }) => { + expected == token + } + _ => false, + }; + if !phase_allowed { + return Err(pool_error(format!( + "refusing pool cleanup for runtime slot {} with persisted phase {:?}", + slot.ownership.instance_id, persisted.phase + ))); + } + slot.ownership.backend_ownership = strongest_backend_ownership( + slot.ownership.backend_ownership, + persisted.backend_ownership, + ); + slot.ownership.storage_owned |= persisted.storage_owned; + } + slot.ownership.phase = RuntimeSlotPhase::PoolCleanup; + if !cleanup_committed { + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + + if let Some(backend) = slot.backend.as_ref() { + backend.kill().await?; + slot.backend = None; + slot.ownership.backend_ownership = BackendOwnership::Stopped; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } else if matches!( + slot.ownership.backend_ownership, + BackendOwnership::Unknown | BackendOwnership::Starting | BackendOwnership::Running + ) { + let spawner = self.spawners.get(slot.ownership.backend).ok_or_else(|| { + pool_error(format!( + "no cleanup spawner registered for {}", + slot.ownership.backend + )) + })?; + spawner + .cleanup_orphan(slot.ownership.instance_id, &slot.run_dir) + .await?; + slot.ownership.backend_ownership = BackendOwnership::Stopped; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + + if slot.ownership.storage_owned { + self.storage + .release_by_id(&slot.ownership.instance_id.to_string()) + .await?; + slot.ownership.storage_owned = false; + write_ownership(&slot.run_dir, &slot.ownership) + .await + .map_err(pool_error)?; + } + tombstone_pool_slot(&self.runtime_root, slot.ownership.instance_id) + .await + .map_err(pool_error)?; + remove_pool_tombstone(&self.runtime_root, slot.ownership.instance_id) + .await + .map_err(pool_error) + } + + fn abandon_lease(&self, slot: RuntimePoolSlot, authority: PoolCleanupAuthority) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&slot.instance_id); + state + .quarantined + .push_back(CleanupSlot::from_runtime_with_authority(slot, authority)); + drop(state); + self.wake.notify_one(); + } + + fn retain_unresolved(&self, handoff: UnresolvedHandoff) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&handoff.slot.instance_id); + state.unresolved.push_back(handoff); + drop(state); + self.wake.notify_one(); + } + + fn complete_lease(&self, instance_id: Uuid) { + let mut state = match self.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.leased.remove(&instance_id); + drop(state); + self.wake.notify_one(); + } +} + +/// Claim guard whose drop path never loses pool or lifecycle ownership. +pub(crate) struct RuntimePoolLease { + pool: Arc, + slot: Option, + owner: LeaseOwner, + cleanup_authority: PoolCleanupAuthority, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum LeaseOwner { + Pool, + Lifecycle, +} + +impl RuntimePoolLease { + fn new(pool: Arc, slot: RuntimePoolSlot) -> Self { + Self { + pool, + slot: Some(slot), + owner: LeaseOwner::Pool, + cleanup_authority: PoolCleanupAuthority::Ready, + } + } + + pub(crate) fn slot(&self) -> Result<&RuntimePoolSlot> { + self.slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot")) + } + + pub(crate) async fn begin_handoff(&mut self, token: Uuid) -> Result<()> { + let slot = self + .slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + let mut ownership = read_ownership(&slot.run_dir, slot.instance_id) + .await + .map_err(pool_error)?; + if ownership.phase != RuntimeSlotPhase::Ready { + return Err(pool_error(format!( + "runtime slot {} is not ready for handoff", + slot.instance_id + ))); + } + ownership.phase = RuntimeSlotPhase::Handoff { token }; + match write_ownership(&slot.run_dir, &ownership).await { + Ok(()) => { + self.cleanup_authority = PoolCleanupAuthority::Handoff(token); + Ok(()) + } + Err(error) => { + if read_ownership(&slot.run_dir, slot.instance_id) + .await + .is_ok_and(|persisted| persisted == ownership) + { + self.cleanup_authority = PoolCleanupAuthority::Handoff(token); + } + Err(pool_error(error)) + } + } + } + + /// Switch the synchronous cancellation fallback after lifecycle is durable. + pub(crate) fn transfer_to_lifecycle(&mut self) { + self.owner = LeaseOwner::Lifecycle; + } + + pub(crate) async fn finish_handoff(&mut self, token: Uuid) -> Result<()> { + let slot = self + .slot + .as_ref() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + finish_ownership_handoff( + &slot.run_dir, + slot.instance_id, + slot.runtime_backend, + slot.backend_ownership, + token, + ) + .await + .map_err(pool_error) + } + + pub(crate) fn into_slot(mut self) -> Result { + let slot = self + .slot + .take() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + self.pool.complete_lease(slot.instance_id); + Ok(slot) + } + + /// Keep an ambiguous handoff visible and counted without choosing a + /// cleanup owner in the current process. + pub(crate) fn retain_unresolved(mut self, token: Uuid, reason: String) -> Result<()> { + if !matches!( + self.cleanup_authority, + PoolCleanupAuthority::Handoff(expected) if expected == token + ) { + return Err(pool_error( + "runtime pool lease has no matching handoff authority", + )); + } + let slot = self + .slot + .take() + .ok_or_else(|| pool_error("runtime pool lease lost its slot"))?; + self.pool.retain_unresolved(UnresolvedHandoff { + slot, + token, + reason, + }); + Ok(()) + } + + fn quarantine(mut self) { + if let Some(slot) = self.slot.take() { + self.pool.abandon_lease(slot, self.cleanup_authority); + } + } +} + +impl Drop for RuntimePoolLease { + fn drop(&mut self) { + let Some(slot) = self.slot.take() else { + return; + }; + match self.owner { + LeaseOwner::Pool => self.pool.abandon_lease(slot, self.cleanup_authority), + LeaseOwner::Lifecycle => self.pool.complete_lease(slot.instance_id), + } + } +} + +struct BuildGuard { + pool: Arc, + ownership: RuntimeSlotOwnership, + storage: Option, + backend: Option, + run_dir: PathBuf, + armed: bool, +} + +impl BuildGuard { + fn new(pool: Arc, run_dir: PathBuf, ownership: RuntimeSlotOwnership) -> Self { + let mut state = match pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.building.insert(ownership.instance_id); + drop(state); + Self { + pool, + ownership, + storage: None, + backend: None, + run_dir, + armed: true, + } + } + + async fn persist(&self) -> Result<()> { + write_ownership(&self.run_dir, &self.ownership) + .await + .map_err(pool_error) + } + + fn publish(mut self, generation: u64) -> Result<()> { + let storage = self + .storage + .take() + .ok_or_else(|| pool_error("completed runtime slot has no storage owner"))?; + let slot = RuntimePoolSlot { + instance_id: self.ownership.instance_id, + storage, + backend: self.backend.take(), + run_dir: self.run_dir.clone(), + runtime_backend: self.ownership.backend, + backend_ownership: self.ownership.backend_ownership, + ready_at: Instant::now(), + }; + let mut state = self + .pool + .state + .lock() + .map_err(|_| pool_error("runtime pool state lock poisoned"))?; + state.building.remove(&slot.instance_id); + if !state.shutting_down + && state.generation == generation + && state.physical_count() < self.pool.target + { + state.ready.push_back(slot); + } else { + state.quarantined.push_back(CleanupSlot::from_runtime(slot)); + } + self.armed = false; + drop(state); + self.pool.wake.notify_one(); + Ok(()) + } +} + +impl Drop for BuildGuard { + fn drop(&mut self) { + if !self.armed { + return; + } + let cleanup = CleanupSlot { + ownership: RuntimeSlotOwnership { + version: self.ownership.version, + instance_id: self.ownership.instance_id, + backend: self.ownership.backend, + backend_ownership: self.ownership.backend_ownership, + storage_owned: self.ownership.storage_owned, + phase: self.ownership.phase, + }, + backend: self.backend.take(), + run_dir: self.run_dir.clone(), + authority: PoolCleanupAuthority::Build, + }; + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.building.remove(&cleanup.ownership.instance_id); + state.quarantined.push_back(cleanup); + drop(state); + self.pool.wake.notify_one(); + } +} + +struct CleanupGuard { + pool: Arc, + slot: Option, +} + +impl CleanupGuard { + fn new(pool: Arc, slot: CleanupSlot) -> Self { + Self { + pool, + slot: Some(slot), + } + } + + fn slot_mut(&mut self) -> Result<&mut CleanupSlot> { + self.slot + .as_mut() + .ok_or_else(|| pool_error("runtime pool cleanup guard lost its slot")) + } + + fn complete(mut self) { + self.slot.take(); + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.cleanup_pending = state.cleanup_pending.saturating_sub(1); + state.consecutive_cleanup_failures = 0; + drop(state); + self.pool.wake.notify_one(); + } + + fn retry(mut self) { + self.retain(); + } + + fn retain(&mut self) { + let Some(slot) = self.slot.take() else { + return; + }; + let mut state = match self.pool.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + state.cleanup_pending = state.cleanup_pending.saturating_sub(1); + state.quarantined.push_back(slot); + drop(state); + self.pool.wake.notify_one(); + } +} + +impl Drop for CleanupGuard { + fn drop(&mut self) { + self.retain(); + } +} + +enum PoolAction { + Build(u64), + Cleanup(CleanupSlot), + Idle, +} + +enum Maintenance { + Progress, + Idle, + RetryAfter(Duration), +} + +fn build_retry_delay(consecutive_failures: u32) -> Duration { + let exponent = consecutive_failures.saturating_sub(1).min(9); + RETRY_BASE_DELAY + .saturating_mul(1_u32 << exponent) + .min(RETRY_MAX_DELAY) +} + +fn strongest_backend_ownership( + left: BackendOwnership, + right: BackendOwnership, +) -> BackendOwnership { + use BackendOwnership::{NotStarted, Running, Starting, Stopped, Unknown}; + + if matches!(left, Unknown) || matches!(right, Unknown) { + Unknown + } else if matches!(left, Running) || matches!(right, Running) { + Running + } else if matches!(left, Starting) || matches!(right, Starting) { + Starting + } else if matches!(left, Stopped) || matches!(right, Stopped) { + Stopped + } else { + NotStarted + } +} + +fn pool_error(message: impl Into) -> BlazeError { + BlazeError::StorageError { + msg: message.into(), + } +} + +#[cfg(test)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct RuntimePoolStatus { + pub(crate) ready: usize, + pub(crate) building: usize, + pub(crate) leased: usize, + pub(crate) quarantined: usize, + pub(crate) unresolved: usize, + pub(crate) cleanup_pending: usize, + pub(crate) capacity: usize, + pub(crate) deficit: usize, +} + +#[cfg(test)] +impl RuntimePoolStatus { + fn from_state(capacity: usize, state: &PoolState) -> Self { + Self { + ready: state.ready.len(), + building: state.building.len(), + leased: state.leased.len(), + quarantined: state.quarantined.len(), + unresolved: state.unresolved.len(), + cleanup_pending: state.cleanup_pending, + capacity, + deficit: capacity.saturating_sub(state.physical_count()), + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::Path; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use async_trait::async_trait; + use blaze_core::storage::{PoolStatus, StorageAcquireError}; + + use crate::spawner::{ + BackendInstance, BackendSpawner, SpawnFailure, SpawnResult, SpawnerRegistry, + }; + + use super::*; + + struct RecordingStorage { + root: PathBuf, + acquire_count: AtomicUsize, + release_started: AtomicUsize, + release_completed: AtomicUsize, + release_delay: Duration, + fail_with_residual_once: AtomicBool, + pending_after_ownership_once: AtomicBool, + owned: Mutex>, + } + + struct RetainedBackend { + kill_count: Arc, + } + + #[async_trait] + impl BackendInstance for RetainedBackend { + fn backend(&self) -> BackendKind { + BackendKind::Mock + } + + async fn try_wait(&self) -> Result> { + Ok(None) + } + + async fn kill(&self) -> Result<()> { + self.kill_count.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + } + + struct ResidualSpawner { + kill_count: Arc, + } + + #[async_trait] + impl BackendSpawner for ResidualSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::with_owner( + pool_error("test spawn retained backend owner"), + Arc::new(RetainedBackend { + kill_count: self.kill_count.clone(), + }), + )) + } + + async fn probe(&self, _binary_path: &Path) -> Result { + Ok(true) + } + + async fn cleanup_orphan(&self, _instance_id: Uuid, _run_dir: &Path) -> Result<()> { + Err(pool_error( + "test retained backend must be cleaned through its handle", + )) + } + } + + impl RecordingStorage { + fn new(root: PathBuf) -> Self { + Self { + root, + acquire_count: AtomicUsize::new(0), + release_started: AtomicUsize::new(0), + release_completed: AtomicUsize::new(0), + release_delay: Duration::ZERO, + fail_with_residual_once: AtomicBool::new(false), + pending_after_ownership_once: AtomicBool::new(false), + owned: Mutex::new(BTreeSet::new()), + } + } + + fn with_release_delay(root: PathBuf, release_delay: Duration) -> Self { + Self { + release_delay, + ..Self::new(root) + } + } + + fn with_residual_failure(root: PathBuf) -> Self { + Self { + fail_with_residual_once: AtomicBool::new(true), + ..Self::new(root) + } + } + + fn with_pending_acquire(root: PathBuf) -> Self { + Self { + pending_after_ownership_once: AtomicBool::new(true), + ..Self::new(root) + } + } + + fn slot(&self, instance_id: &str) -> StorageSlot { + let instance_dir = self.root.join(instance_id); + StorageSlot { + id: instance_id.to_string(), + rootfs_path: instance_dir.join("rootfs"), + mem_path: instance_dir.join("memory"), + mem_diff_path: instance_dir.join("memory.diff"), + rootfs_diff_path: instance_dir.join("rootfs.diff"), + instance_dir, + } + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> Result { + Ok(true) + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.acquire_count.fetch_add(1, Ordering::SeqCst); + let slot = self.slot(&opts.instance_id); + tokio::fs::create_dir_all(&slot.instance_dir) + .await + .map_err(BlazeError::from)?; + self.owned + .lock() + .map_err(|_| StorageAcquireError::clean(pool_error("test storage lock poisoned")))? + .insert(opts.instance_id.clone()); + if self + .pending_after_ownership_once + .swap(false, Ordering::SeqCst) + { + std::future::pending::<()>().await; + } + if self.fail_with_residual_once.swap(false, Ordering::SeqCst) { + return Err(StorageAcquireError::with_residual( + pool_error("test acquire retained residual storage"), + slot, + )); + } + Ok(slot) + } + + async fn release(&self, slot: StorageSlot) -> Result<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> Result<()> { + self.release_started.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(self.release_delay).await; + self.owned + .lock() + .map_err(|_| pool_error("test storage lock poisoned"))? + .remove(instance_id); + self.release_completed.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + fn supports_runtime_pool_recovery(&self) -> bool { + true + } + + async fn list_owned_ids(&self) -> Result> { + Ok(self + .owned + .lock() + .map_err(|_| pool_error("test storage lock poisoned"))? + .iter() + .cloned() + .collect()) + } + + async fn reconstruct(&self, instance_id: &str) -> Result { + Ok(self.slot(instance_id)) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> Result<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> Result { + Ok(0) + } + } + + fn prototype(warm_ttl: Duration) -> PoolPrototype { + PoolPrototype { + image_digest: "sha256:test".to_string(), + policy_name: "test-policy".to_string(), + workload_class: WorkloadClass::AgentTool, + templates: Vec::new(), + kernel_hooks: Vec::new(), + binary_path: PathBuf::from("/unused"), + runtime_backend: BackendKind::Mock, + backend: BackendConfigs::default(), + vm: None, + warm_ttl, + } + } + + fn make_pool( + target: usize, + runtime_root: &Path, + storage: Arc, + cancellation: CancellationToken, + ) -> Arc { + std::fs::create_dir_all(runtime_root).expect("runtime root"); + RuntimeWarmPool::new( + target, + false, + 1024, + 1024, + runtime_root.to_path_buf(), + storage, + Arc::new(SpawnerRegistry::new()), + Duration::from_secs(60), + Duration::from_secs(3600), + cancellation, + ) + .expect("runtime pool") + } + + fn slot(storage: &RecordingStorage, runtime_root: &Path, age: Duration) -> RuntimePoolSlot { + let instance_id = Uuid::new_v4(); + RuntimePoolSlot { + instance_id, + storage: storage.slot(&instance_id.to_string()), + backend: None, + run_dir: runtime_root.join(instance_id.to_string()), + runtime_backend: BackendKind::Mock, + backend_ownership: BackendOwnership::NotStarted, + ready_at: Instant::now() + .checked_sub(age) + .expect("slot age must fit in test clock"), + } + } + + async fn persist_ready_slot(slot: &RuntimePoolSlot) { + std::fs::create_dir_all(&slot.run_dir).expect("slot run directory"); + let mut ownership = RuntimeSlotOwnership::new(slot.instance_id, slot.runtime_backend); + ownership.storage_owned = true; + ownership.backend_ownership = slot.backend_ownership; + ownership.phase = RuntimeSlotPhase::Ready; + write_ownership(&slot.run_dir, &ownership) + .await + .expect("ready ownership"); + } + + async fn wait_for_ready(pool: &RuntimeWarmPool) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if pool.status().ready != 0 { + return; + } + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("runtime slot becomes ready"); + } + + #[tokio::test] + async fn zero_capacity_never_starts_worker_or_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 0, + &temp.path().join("runtime"), + storage.clone(), + CancellationToken::new(), + ); + + assert!( + !pool + .configure(prototype(Duration::from_secs(60))) + .expect("configure disabled pool") + ); + tokio::task::yield_now().await; + + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!( + pool.status(), + RuntimePoolStatus { + ready: 0, + building: 0, + leased: 0, + quarantined: 0, + unresolved: 0, + cleanup_pending: 0, + capacity: 0, + deficit: 0, + } + ); + } + + #[tokio::test] + async fn non_prefork_build_persists_ready_slot_before_claim() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage.clone(), + CancellationToken::new(), + ); + + assert!( + pool.configure(prototype(Duration::from_secs(60))) + .expect("configure pool") + ); + wait_for_ready(&pool).await; + + let (instance_id, run_dir) = { + let state = pool.state.lock().expect("pool state"); + let slot = state.ready.front().expect("ready slot"); + (slot.instance_id, slot.run_dir.clone()) + }; + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("read ready ownership"); + assert_eq!(ownership.phase, RuntimeSlotPhase::Ready); + assert!(ownership.storage_owned); + assert_eq!(ownership.backend_ownership, BackendOwnership::NotStarted); + + let lease = pool + .acquire() + .await + .expect("claim ready slot") + .expect("available ready slot"); + assert_eq!(lease.slot().expect("leased slot").instance_id, instance_id); + assert_eq!(pool.status().leased, 1); + + pool.begin_shutdown(); + drop(lease); + pool.shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect("shutdown pool"); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn cleanup_does_not_overwrite_a_transferred_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + let owner_token = Uuid::new_v4(); + let mut transferred = read_ownership(&runtime.run_dir, runtime.instance_id) + .await + .expect("ready ownership"); + transferred.phase = RuntimeSlotPhase::LifecycleOwned { token: owner_token }; + write_ownership(&runtime.run_dir, &transferred) + .await + .expect("transferred ownership"); + let mut cleanup = CleanupSlot::from_runtime(runtime); + + let error = pool + .cleanup_slot(&mut cleanup) + .await + .expect_err("stale pool owner must not clean lifecycle resources"); + + assert!(error.to_string().contains("refusing pool cleanup")); + assert_eq!( + read_ownership(&cleanup.run_dir, cleanup.ownership.instance_id) + .await + .expect("ownership remains") + .phase, + RuntimeSlotPhase::LifecycleOwned { token: owner_token } + ); + assert_eq!(storage.release_started.load(Ordering::SeqCst), 0); + assert!( + storage + .owned + .lock() + .expect("owned storage") + .contains(&cleanup.ownership.instance_id.to_string()) + ); + } + + #[tokio::test] + async fn matching_handoff_can_return_to_pool_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + let instance_id = runtime.instance_id; + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(instance_id.to_string()); + pool.state + .lock() + .expect("pool state") + .ready + .push_back(runtime); + let mut lease = pool + .acquire() + .await + .expect("acquire slot") + .expect("ready slot"); + let owner_token = Uuid::new_v4(); + lease + .begin_handoff(owner_token) + .await + .expect("begin handoff"); + + drop(lease); + let mut cleanup = pool + .state + .lock() + .expect("pool state") + .quarantined + .pop_front() + .expect("abandoned handoff"); + pool.cleanup_slot(&mut cleanup) + .await + .expect("matching handoff cleanup"); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!( + !storage + .owned + .lock() + .expect("owned storage") + .contains(&instance_id.to_string()) + ); + assert!(!runtime_root.join(instance_id.to_string()).exists()); + } + + #[tokio::test] + async fn cleanup_retry_recommits_intent_before_release() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let runtime = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + let mut cleanup = CleanupSlot::from_runtime(runtime); + cleanup.ownership.phase = RuntimeSlotPhase::PoolCleanup; + let instance_id = cleanup.ownership.instance_id; + let run_dir = cleanup.run_dir.clone(); + + let cleanup_pool = pool.clone(); + let cleanup_task = + tokio::spawn(async move { cleanup_pool.cleanup_slot(&mut cleanup).await }); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("release starts after cleanup intent is durable"); + cleanup_task.abort(); + assert!( + cleanup_task + .await + .expect_err("cleanup is cancelled during release") + .is_cancelled() + ); + + assert_eq!(storage.release_started.load(Ordering::SeqCst), 1); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("cleanup journal") + .phase, + RuntimeSlotPhase::PoolCleanup + ); + } + + #[tokio::test] + async fn shutdown_reports_a_panicked_worker() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage, CancellationToken::new()); + *pool.worker.lock().expect("worker lock") = Some(tokio::spawn(async { + panic!("injected runtime pool worker panic"); + })); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect_err("worker panic must fail shutdown"); + + assert!( + error + .to_string() + .contains("runtime pool worker failed while stopping") + ); + } + + #[tokio::test] + async fn every_owned_state_consumes_physical_capacity() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(5, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let quarantined = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&quarantined).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(quarantined.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state.ready.push_back(ready); + state.building.insert(Uuid::new_v4()); + state.leased.insert(Uuid::new_v4()); + state + .quarantined + .push_back(CleanupSlot::from_runtime(quarantined)); + state.cleanup_pending = 1; + } + + let status = pool.status(); + assert_eq!(status.ready, 1); + assert_eq!(status.building, 1); + assert_eq!(status.leased, 1); + assert_eq!(status.quarantined, 1); + assert_eq!(status.cleanup_pending, 1); + assert_eq!(status.capacity, 5); + assert_eq!(status.deficit, 0); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + assert!(matches!( + pool.maintain_once().await.expect("maintain full pool"), + Maintenance::Progress + )); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn worker_start_rechecks_shutdown_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage, + CancellationToken::new(), + ); + + pool.begin_shutdown(); + let error = pool + .ensure_worker() + .expect_err("worker must not start after shutdown wins the race"); + + assert!(error.to_string().contains("shutting down")); + assert!(!pool.has_tracked_worker()); + } + + #[tokio::test] + async fn expired_shutdown_wait_still_stops_new_work() { + let temp = tempfile::tempdir().expect("tempdir"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool( + 1, + &temp.path().join("runtime"), + storage, + CancellationToken::new(), + ); + let _held_shutdown = pool.shutdown.lock().await; + + let error = pool + .shutdown_until(Instant::now()) + .await + .expect_err("expired coordination wait must fail"); + + assert!(error.to_string().contains("coordination")); + assert!(pool.cancellation.is_cancelled()); + assert!(pool.state.lock().expect("pool state").shutting_down); + assert!( + pool.ensure_worker().is_err(), + "expired shutdown still prevents later worker creation" + ); + } + + #[tokio::test] + async fn expired_slot_is_quarantined_instead_of_claimed() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + { + let mut state = pool.state.lock().expect("pool state"); + state + .ready + .push_back(slot(&storage, &runtime_root, Duration::from_secs(61))); + } + + assert!(pool.acquire().await.expect("claim expired slot").is_none()); + let status = pool.status(); + assert_eq!(status.ready, 0); + assert_eq!(status.leased, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + } + + #[tokio::test] + async fn residual_acquire_failure_remains_owned_until_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_residual_failure( + temp.path().join("storage"), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + pool.state.lock().expect("pool state").prototype = Some(prototype(Duration::from_secs(60))); + + pool.build_slot(0) + .await + .expect_err("acquire reports residual owner"); + + let status = pool.status(); + assert_eq!(status.building, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + assert_eq!(storage.owned.lock().expect("owned storage").len(), 1); + + assert!(matches!( + pool.maintain_once().await.expect("cleanup residual"), + Maintenance::Progress + )); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().quarantined, 0); + } + + #[tokio::test] + async fn shutdown_cleans_storage_from_a_cancelled_acquire() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_pending_acquire( + temp.path().join("storage"), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + pool.configure(prototype(Duration::from_secs(60))) + .expect("configure pool"); + let instance_id = tokio::time::timeout(Duration::from_secs(1), async { + loop { + if let Some(instance_id) = storage + .owned + .lock() + .expect("owned storage") + .iter() + .next() + .cloned() + { + return Uuid::parse_str(&instance_id).expect("owned UUID"); + } + tokio::task::yield_now().await; + } + }) + .await + .expect("provider creates an owner before acquire returns"); + let ownership = read_ownership(&runtime_root.join(instance_id.to_string()), instance_id) + .await + .expect("conservative storage journal"); + assert!(ownership.storage_owned); + assert_eq!(ownership.phase, RuntimeSlotPhase::Building); + + pool.shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect("shutdown cancels acquire and cleans the owner"); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().building, 0); + assert_eq!(pool.status().quarantined, 0); + assert!(!runtime_root.join(instance_id.to_string()).exists()); + } + + #[tokio::test] + async fn residual_prefork_failure_remains_owned_until_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let kill_count = Arc::new(AtomicUsize::new(0)); + let mut registry = SpawnerRegistry::new(); + registry.insert( + BackendKind::Mock, + Arc::new(ResidualSpawner { + kill_count: kill_count.clone(), + }), + ); + let pool = RuntimeWarmPool::new( + 1, + true, + 1024, + 1024, + runtime_root, + storage.clone(), + Arc::new(registry), + Duration::from_secs(60), + Duration::from_secs(3600), + CancellationToken::new(), + ) + .expect("runtime pool"); + pool.state.lock().expect("pool state").prototype = Some(prototype(Duration::from_secs(60))); + + pool.build_slot(0) + .await + .expect_err("prefork spawn reports residual owner"); + + let status = pool.status(); + assert_eq!(status.building, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(status.deficit, 0); + assert_eq!(kill_count.load(Ordering::SeqCst), 0); + + assert!(matches!( + pool.maintain_once().await.expect("cleanup residual"), + Maintenance::Progress + )); + assert_eq!(kill_count.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + assert_eq!(pool.status().quarantined, 0); + } + + #[tokio::test] + async fn unresolved_handoff_counts_capacity_and_blocks_shutdown() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::new(temp.path().join("storage"))); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let instance_id = ready.instance_id; + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state.ready.push_back(ready); + } + + let mut lease = pool + .acquire() + .await + .expect("acquire slot") + .expect("ready slot"); + let token = Uuid::new_v4(); + lease.begin_handoff(token).await.expect("begin handoff"); + lease + .retain_unresolved(token, "test publication is ambiguous".to_string()) + .expect("retain ambiguous handoff"); + + let status = pool.status(); + assert_eq!(status.leased, 0); + assert_eq!(status.unresolved, 1); + assert_eq!(status.deficit, 0); + assert!(matches!( + pool.maintain_once().await.expect("maintain bounded pool"), + Maintenance::Idle + )); + assert_eq!(storage.acquire_count.load(Ordering::SeqCst), 0); + let ownership = read_ownership(&runtime_root.join(instance_id.to_string()), instance_id) + .await + .expect("retained handoff journal"); + assert_eq!(ownership.phase, RuntimeSlotPhase::Handoff { token }); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_secs(1)) + .await + .expect_err("unresolved owner must be reported"); + assert!(error.to_string().contains(&instance_id.to_string())); + assert!( + error + .to_string() + .contains("unresolved lifecycle publication") + ); + assert!(error.to_string().contains("test publication is ambiguous")); + assert_eq!(pool.status().unresolved, 1); + assert!( + storage + .owned + .lock() + .expect("owned storage") + .contains(&instance_id.to_string()) + ); + } + + #[tokio::test] + async fn cancelled_shutdown_retains_active_and_unstarted_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_millis(500), + )); + let pool = make_pool(2, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + let quarantined = slot(&storage, &runtime_root, Duration::ZERO); + let expected = BTreeSet::from([ready.instance_id, quarantined.instance_id]); + for runtime in [&ready, &quarantined] { + persist_ready_slot(runtime).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(runtime.instance_id.to_string()); + } + { + let mut state = pool.state.lock().expect("pool state"); + state.ready.push_back(ready); + state + .quarantined + .push_back(CleanupSlot::from_runtime(quarantined)); + } + + let shutdown_pool = pool.clone(); + let shutdown = tokio::spawn(async move { + shutdown_pool + .shutdown_until(Instant::now() + Duration::from_secs(30)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown starts one cleanup"); + + shutdown.abort(); + assert!( + shutdown + .await + .expect_err("shutdown task is cancelled") + .is_cancelled() + ); + + let status = pool.status(); + assert_eq!(status.ready, 1); + assert_eq!(status.quarantined, 1); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.deficit, 0); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert_eq!(storage.owned.lock().expect("owned storage").len(), 2); + let retained = { + let state = pool.state.lock().expect("pool state"); + state + .ready + .iter() + .map(|slot| slot.instance_id) + .chain( + state + .quarantined + .iter() + .map(|slot| slot.ownership.instance_id), + ) + .collect::>() + }; + assert_eq!(retained, expected); + + pool.shutdown_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry retained shutdown cleanup"); + let status = pool.status(); + assert_eq!(status.ready, 0); + assert_eq!(status.quarantined, 0); + assert_eq!(status.cleanup_pending, 0); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn shutdown_joins_worker_and_shares_one_deadline() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_millis(40), + )); + let cancellation = CancellationToken::new(); + let pool = make_pool(2, &runtime_root, storage.clone(), cancellation.clone()); + for _ in 0..2 { + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + pool.state + .lock() + .expect("pool state") + .ready + .push_back(ready); + } + + let worker_joined = Arc::new(AtomicBool::new(false)); + let joined = worker_joined.clone(); + *pool.worker.lock().expect("worker lock") = Some(tokio::spawn(async move { + cancellation.cancelled().await; + joined.store(true, Ordering::SeqCst); + })); + + let start = Instant::now(); + let deadline = start + Duration::from_millis(60); + let error = pool + .shutdown_until(deadline) + .await + .expect_err("second cleanup must share the first deadline"); + + assert!(error.to_string().contains("shared shutdown deadline")); + assert_eq!(Instant::now(), deadline); + assert!(worker_joined.load(Ordering::SeqCst)); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!(storage.release_started.load(Ordering::SeqCst), 2); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert_eq!(pool.status().quarantined, 1); + } + + #[tokio::test] + async fn aborted_worker_retains_its_active_cleanup_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let error = pool + .shutdown_until(Instant::now() + Duration::from_millis(10)) + .await + .expect_err("active cleanup exceeds shutdown deadline"); + + assert!(error.to_string().contains("shared shutdown deadline")); + let status = pool.status(); + assert_eq!(status.cleanup_pending, 0); + assert_eq!(status.quarantined, 1); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + } + + #[tokio::test] + async fn cancelled_shutdown_retains_worker_for_a_joining_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let shutdown_pool = pool.clone(); + let shutdown = tokio::spawn(async move { + shutdown_pool + .shutdown_until(Instant::now() + Duration::from_secs(10)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while pool.worker.lock().expect("worker lock").is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("shutdown takes worker handle"); + shutdown.abort(); + assert!( + shutdown + .await + .expect_err("outer shutdown is cancelled") + .is_cancelled() + ); + + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 0); + assert!( + pool.worker.lock().expect("worker lock").is_some(), + "cancelled shutdown must retain the aborted worker handle" + ); + + pool.shutdown_until(Instant::now() + Duration::from_secs(2)) + .await + .expect("retry joins the worker and releases the retained owner"); + let status = pool.status(); + assert_eq!(status.quarantined, 0); + assert_eq!(status.cleanup_pending, 0); + assert!(pool.worker.lock().expect("worker lock").is_none()); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } + + #[tokio::test] + async fn concurrent_shutdown_waiter_joins_a_worker_retained_by_cancellation() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime"); + let storage = Arc::new(RecordingStorage::with_release_delay( + temp.path().join("storage"), + Duration::from_secs(1), + )); + let pool = make_pool(1, &runtime_root, storage.clone(), CancellationToken::new()); + let ready = slot(&storage, &runtime_root, Duration::ZERO); + persist_ready_slot(&ready).await; + storage + .owned + .lock() + .expect("owned storage") + .insert(ready.instance_id.to_string()); + { + let mut state = pool.state.lock().expect("pool state"); + state.prototype = Some(prototype(Duration::from_secs(60))); + state + .quarantined + .push_back(CleanupSlot::from_runtime(ready)); + } + pool.ensure_worker().expect("start worker"); + pool.wake.notify_one(); + tokio::time::timeout(Duration::from_secs(1), async { + while storage.release_started.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("worker starts cleanup"); + + let first_pool = pool.clone(); + let first = tokio::spawn(async move { + first_pool + .shutdown_until(Instant::now() + Duration::from_secs(10)) + .await + }); + tokio::time::timeout(Duration::from_secs(1), async { + while pool.has_tracked_worker() { + tokio::task::yield_now().await; + } + }) + .await + .expect("first shutdown takes worker handle"); + let second_pool = pool.clone(); + let second = tokio::spawn(async move { + second_pool + .shutdown_until(Instant::now() + Duration::from_secs(3)) + .await + }); + tokio::task::yield_now().await; + + first.abort(); + assert!( + first + .await + .expect_err("first shutdown is cancelled") + .is_cancelled() + ); + tokio::time::timeout(Duration::from_secs(3), second) + .await + .expect("second shutdown finishes") + .expect("second shutdown task") + .expect("second shutdown joins and cleans the retained worker"); + + assert!(!pool.has_tracked_worker()); + assert_eq!(pool.status().quarantined, 0); + assert_eq!(pool.status().cleanup_pending, 0); + assert_eq!(storage.release_completed.load(Ordering::SeqCst), 1); + assert!(storage.owned.lock().expect("owned storage").is_empty()); + } +} diff --git a/src/blaze/crates/blazed/src/runtime_pool/recovery.rs b/src/blaze/crates/blazed/src/runtime_pool/recovery.rs new file mode 100644 index 0000000000..daf8421c04 --- /dev/null +++ b/src/blaze/crates/blazed/src/runtime_pool/recovery.rs @@ -0,0 +1,3660 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable ownership records and startup recovery for unclaimed runtime slots. + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use blaze_core::backend::BackendKind; +use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, RuntimeLocation, SandboxInstance, SandboxState, +}; +use blaze_core::storage::StorageProvider; +use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::SpawnerRegistry; + +const OWNERSHIP_FILE: &str = "ownership.json"; +const OWNERSHIP_VERSION: u32 = 1; +const CLEANUP_NAMESPACE: &str = ".cleanup"; +const DELETION_PROOF_NAMESPACE: &str = ".deletion-proofs"; +const STARTUP_RECONCILE_TIMEOUT: Duration = Duration::from_secs(30); + +#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "kebab-case", tag = "kind")] +pub(super) enum RuntimeSlotPhase { + Building, + Ready, + Handoff { token: Uuid }, + LifecycleOwned { token: Uuid }, + PoolCleanup, + LifecycleCleanup { token: Uuid }, +} + +#[derive(Debug, Clone, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +pub(super) struct RuntimeSlotOwnership { + pub(super) version: u32, + pub(super) instance_id: Uuid, + pub(super) backend: BackendKind, + pub(super) backend_ownership: BackendOwnership, + pub(super) storage_owned: bool, + pub(super) phase: RuntimeSlotPhase, +} + +impl RuntimeSlotOwnership { + pub(super) fn new(instance_id: Uuid, backend: BackendKind) -> Self { + Self { + version: OWNERSHIP_VERSION, + instance_id, + backend, + backend_ownership: BackendOwnership::NotStarted, + storage_owned: false, + phase: RuntimeSlotPhase::Building, + } + } +} + +/// Lifecycle identity used to keep transferred warm resources out of cleanup. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct DurableRuntimeOwner { + instance_id: Uuid, + runtime_location: RuntimeLocation, + backend: BackendKind, + backend_ownership: BackendOwnership, + state: SandboxState, + operation: Option, + runtime_owner_token: Option, + clean_terminal: bool, +} + +impl From<&SandboxInstance> for DurableRuntimeOwner { + fn from(instance: &SandboxInstance) -> Self { + Self { + instance_id: instance.id, + runtime_location: instance.runtime_location, + backend: instance.backend, + backend_ownership: instance.backend_ownership, + state: instance.state, + operation: instance.operation.as_ref().map(|operation| operation.kind), + runtime_owner_token: instance.runtime_owner_token, + clean_terminal: instance.is_clean_terminal(), + } + } +} + +impl DurableRuntimeOwner { + fn is_clean_terminal(&self) -> bool { + self.clean_terminal + } +} + +/// Derive the only accepted backend runtime directory for an owned instance. +pub(crate) fn runtime_dir( + state_dir: &Path, + location: RuntimeLocation, + instance_id: Uuid, +) -> PathBuf { + match location { + RuntimeLocation::Sandbox => state_dir.join(instance_id.to_string()), + RuntimeLocation::WarmPool => state_dir.join("runtime-pool").join(instance_id.to_string()), + } +} + +/// Release runtime slots that were never adopted by durable lifecycle state. +/// +/// A provider-only slot is safe to release by stable ID. Once a runtime +/// directory exists, its ownership record is required so restart recovery can +/// select the original backend instead of guessing from current policy. +pub(crate) async fn reconcile_runtime_slots( + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + reconcile_runtime_slots_until( + tokio::time::Instant::now() + STARTUP_RECONCILE_TIMEOUT, + runtime_root, + durable_owners, + storage, + spawners, + ) + .await +} + +async fn reconcile_runtime_slots_until( + deadline: tokio::time::Instant, + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + match tokio::time::timeout_at( + deadline, + reconcile_runtime_slots_inner(runtime_root, durable_owners, storage, spawners), + ) + .await + { + Ok(result) => result, + Err(_) => Err(recovery_error( + "runtime slot reconciliation exceeded its shared startup deadline".to_string(), + )), + } +} + +async fn reconcile_runtime_slots_inner( + runtime_root: &Path, + durable_owners: &HashMap, + storage: &dyn StorageProvider, + spawners: &SpawnerRegistry, +) -> Result { + ensure_real_directory(runtime_root, "runtime slot root").await?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace").await?; + let deletion_proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &deletion_proof_root).await?; + restore_deletion_proofs(&cleanup_root, &deletion_proof_root).await?; + + let (runtime_dirs, tombstones, root_errors) = + scan_runtime_dirs(runtime_root, &cleanup_root, &deletion_proof_root).await?; + if !root_errors.is_empty() { + return Err(unresolved_items(root_errors)); + } + let mut errors = + validate_durable_owners(runtime_root, durable_owners, &runtime_dirs, &tombstones).await; + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + let mut provider_ids = BTreeSet::new(); + let mut provider_errors = Vec::new(); + if storage.supports_runtime_pool_recovery() { + match storage.list_owned_ids().await { + Ok(ids) => { + for owned_id in ids { + match parse_stable_id(&owned_id) { + Ok(id) => { + provider_ids.insert(id); + } + Err(error) => { + provider_errors + .push(format!("provider-owned slot {owned_id:?}: {error}")); + } + } + } + } + Err(error) => { + provider_errors.push(format!("read provider-owned slot inventory: {error}")); + } + } + } else if !runtime_dirs.is_empty() { + errors.push("storage provider cannot inventory slots for runtime recovery".to_string()); + return Err(unresolved_items(errors)); + } + if !provider_errors.is_empty() { + provider_errors.extend(errors); + return Err(unresolved_items(provider_errors)); + } + for instance_id in tombstones.keys() { + if provider_ids.contains(instance_id) { + errors.push(format!( + "{instance_id}: cleanup tombstone conflicts with provider-owned storage" + )); + } + } + for (instance_id, tombstone) in &tombstones { + if !durable_owners.contains_key(instance_id) { + match read_ownership(tombstone, *instance_id).await { + Ok(ownership) if ownership.phase == RuntimeSlotPhase::PoolCleanup => {} + Ok(_) => errors.push(format!( + "{instance_id}: cleanup tombstone has no durable lifecycle owner and is not \ + pool cleanup" + )), + Err(error) => errors.push(format!("{instance_id}: {error}")), + } + } + } + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + let candidates = runtime_dirs + .keys() + .chain(provider_ids.iter()) + .copied() + .collect::>(); + let mut cleaned = 0; + for (instance_id, tombstone) in tombstones { + if durable_owners.get(&instance_id).is_some_and(|owner| { + !owner.is_clean_terminal() + && owner.runtime_location == RuntimeLocation::WarmPool + && owner.state == SandboxState::Destroyed + && owner.backend_ownership == BackendOwnership::Stopped + && owner.operation == Some(OperationKind::Destroy) + }) { + continue; + } + let expected_owner = match durable_owners.get(&instance_id) { + Some(owner) => match owner.runtime_owner_token { + Some(token) => CleanupOwner::Lifecycle(token), + None => { + errors.push(format!( + "{instance_id}: durable lifecycle owner has no runtime ownership token" + )); + continue; + } + }, + None => CleanupOwner::Pool, + }; + match remove_owned_tombstone(runtime_root, instance_id, expected_owner).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: continue runtime tombstone cleanup {}: {error}", + tombstone.display() + )), + } + } + for instance_id in candidates { + let run_dir = runtime_dirs.get(&instance_id); + if durable_owners + .get(&instance_id) + .is_some_and(|owner| !owner.is_clean_terminal()) + { + continue; + } + + let Some(run_dir) = run_dir else { + match storage.release_by_id(&instance_id.to_string()).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: release provider-only slot: {error}" + )), + } + continue; + }; + + let ownership_path = run_dir.join(OWNERSHIP_FILE); + let ownership_missing = match tokio::fs::symlink_metadata(&ownership_path).await { + Ok(_) => false, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => true, + Err(error) => { + errors.push(format!( + "{instance_id}: inspect ownership journal {}: {error}", + ownership_path.display() + )); + continue; + } + }; + if ownership_missing && !provider_ids.contains(&instance_id) { + match tokio::fs::remove_dir(run_dir).await { + Ok(()) => match sync_directory(runtime_root).await { + Ok(()) => { + cleaned += 1; + continue; + } + Err(error) => { + errors.push(format!( + "{instance_id}: sync runtime slot root after removing an empty \ + unjournaled slot: {error}" + )); + continue; + } + }, + Err(error) => { + errors.push(format!( + "{instance_id}: ownership journal {} is missing and the runtime directory \ + is not safely removable as empty: {error}", + ownership_path.display() + )); + continue; + } + } + } + + let mut ownership = match read_ownership(run_dir, instance_id).await { + Ok(ownership) => ownership, + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + }; + if matches!( + ownership.phase, + RuntimeSlotPhase::LifecycleOwned { .. } | RuntimeSlotPhase::LifecycleCleanup { .. } + ) { + errors.push(format!( + "{instance_id}: runtime slot records lifecycle ownership but durable lifecycle \ + metadata is missing" + )); + continue; + } + if ownership.phase != RuntimeSlotPhase::PoolCleanup { + ownership.phase = RuntimeSlotPhase::PoolCleanup; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record pool cleanup ownership: {error}" + )); + continue; + } + } + + if matches!( + ownership.backend_ownership, + BackendOwnership::Unknown | BackendOwnership::Starting | BackendOwnership::Running + ) { + let Some(spawner) = spawners.get(ownership.backend) else { + errors.push(format!( + "{instance_id}: no recovery spawner registered for recorded backend {}", + ownership.backend + )); + continue; + }; + if let Err(error) = spawner.cleanup_orphan(instance_id, run_dir).await { + errors.push(format!( + "{instance_id}: clean recorded backend {}: {error}", + ownership.backend + )); + continue; + } + ownership.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record completed backend cleanup: {error}" + )); + continue; + } + } + + if ownership.storage_owned || provider_ids.contains(&instance_id) { + if let Err(error) = storage.release_by_id(&instance_id.to_string()).await { + errors.push(format!("{instance_id}: release recorded storage: {error}")); + continue; + } + ownership.storage_owned = false; + if let Err(error) = write_ownership(run_dir, &ownership).await { + errors.push(format!( + "{instance_id}: record completed storage cleanup: {error}" + )); + continue; + } + } + + match tombstone_pool_slot(runtime_root, instance_id).await { + Ok(()) => match remove_pool_tombstone(runtime_root, instance_id).await { + Ok(()) => cleaned += 1, + Err(error) => errors.push(format!( + "{instance_id}: remove runtime cleanup tombstone: {error}" + )), + }, + Err(error) => errors.push(format!( + "{instance_id}: tombstone runtime directory {}: {error}", + run_dir.display() + )), + } + } + + if errors.is_empty() { + Ok(cleaned) + } else { + Err(unresolved_items(errors)) + } +} + +async fn validate_durable_owners( + runtime_root: &Path, + durable_owners: &HashMap, + runtime_dirs: &BTreeMap, + tombstones: &BTreeMap, +) -> Vec { + let mut errors = Vec::new(); + for (instance_id, owner) in durable_owners { + if owner.is_clean_terminal() { + if let Some(run_dir) = runtime_dirs.get(instance_id) { + errors.push(format!( + "{instance_id}: clean terminal lifecycle state still has active runtime slot {}", + run_dir.display() + )); + } + if let Some(tombstone) = tombstones.get(instance_id) + && let Err(error) = + validate_transferred_run_dir(tombstone, *instance_id, owner, true).await + { + errors.push(format!("{instance_id}: {error}")); + } + continue; + } + if owner.instance_id != *instance_id { + errors.push(format!( + "{instance_id}: durable owner map records instance {}", + owner.instance_id + )); + continue; + } + if let Some(tombstone) = tombstones.get(instance_id) { + if owner.runtime_location != RuntimeLocation::WarmPool + || owner.state != SandboxState::Destroyed + || owner.backend_ownership != BackendOwnership::Stopped + || owner.operation != Some(OperationKind::Destroy) + { + errors.push(format!( + "{instance_id}: cleanup tombstone {} collides with durable lifecycle owner", + tombstone.display() + )); + } else if let Err(error) = + validate_transferred_run_dir(tombstone, *instance_id, owner, true).await + { + errors.push(format!("{instance_id}: {error}")); + } + continue; + } + match owner.runtime_location { + RuntimeLocation::Sandbox => { + if let Some(run_dir) = runtime_dirs.get(instance_id) { + errors.push(format!( + "{instance_id}: warm-slot directory {} collides with sandbox runtime owner \ + in {} state", + run_dir.display(), + owner.state + )); + } + } + RuntimeLocation::WarmPool => match runtime_dirs.get(instance_id) { + Some(run_dir) => { + if let Err(error) = + validate_transferred_run_dir(run_dir, *instance_id, owner, false).await + { + errors.push(format!("{instance_id}: {error}")); + } + } + None => errors.push(format!( + "{instance_id}: durable warm-slot owner in {} state is missing canonical \ + runtime directory {}", + owner.state, + runtime_root.join(instance_id.to_string()).display() + )), + }, + } + } + errors +} + +async fn validate_transferred_run_dir( + run_dir: &Path, + instance_id: Uuid, + owner: &DurableRuntimeOwner, + tombstoned: bool, +) -> std::result::Result<(), String> { + let ownership_path = run_dir.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&ownership_path).await { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Err(format!( + "durable warm-slot owner is missing ownership journal {}", + ownership_path.display() + )); + } + Err(error) => { + return Err(format!( + "inspect transferred ownership {}: {error}", + ownership_path.display() + )); + } + } + let ownership = read_ownership(run_dir, instance_id).await?; + if ownership.backend != owner.backend { + return Err(format!( + "{} records backend {} but durable lifecycle owns {}", + ownership_path.display(), + ownership.backend, + owner.backend + )); + } + let Some(owner_token) = owner.runtime_owner_token else { + return Err(format!( + "durable warm-slot owner has no runtime ownership token for {}", + ownership_path.display() + )); + }; + let current_phase = ownership.phase; + match current_phase { + RuntimeSlotPhase::Handoff { token } => { + if tombstoned { + return Err(format!( + "{} is tombstoned before its ownership handoff completed", + ownership_path.display() + )); + } + if token != owner_token { + return Err(format!( + "{} records handoff token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + if ownership.backend_ownership != owner.backend_ownership { + return Err(format!( + "{} records backend ownership {:?} but durable lifecycle records {:?}", + ownership_path.display(), + ownership.backend_ownership, + owner.backend_ownership + )); + } + if !ownership.storage_owned { + return Err(format!( + "{} records released storage during ownership handoff", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::LifecycleOwned { token } => { + if tombstoned { + return Err(format!( + "{} is tombstoned without a lifecycle cleanup phase", + ownership_path.display() + )); + } + if token != owner_token { + return Err(format!( + "{} records lifecycle token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::LifecycleCleanup { token } => { + if token != owner_token { + return Err(format!( + "{} records cleanup token {token} but durable lifecycle records {owner_token}", + ownership_path.display() + )); + } + if owner.operation != Some(OperationKind::Destroy) && !owner.is_clean_terminal() { + return Err(format!( + "{} records lifecycle cleanup without a durable destroy operation", + ownership_path.display() + )); + } + } + RuntimeSlotPhase::Building | RuntimeSlotPhase::Ready | RuntimeSlotPhase::PoolCleanup => { + return Err(format!( + "{} remains pool-owned while durable lifecycle metadata exists", + ownership_path.display() + )); + } + } + if matches!( + current_phase, + RuntimeSlotPhase::Handoff { .. } | RuntimeSlotPhase::LifecycleOwned { .. } + ) && !ownership.storage_owned + { + return Err(format!( + "{} records lifecycle ownership without owned storage", + ownership_path.display() + )); + } + Ok(()) +} + +async fn scan_runtime_dirs( + runtime_root: &Path, + cleanup_root: &Path, + deletion_proof_root: &Path, +) -> Result<( + BTreeMap, + BTreeMap, + Vec, +)> { + let mut directories = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(runtime_root).await.map_err(|error| { + recovery_error(format!( + "read runtime slot root {}: {error}", + runtime_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime slot entry under {}: {error}", + runtime_root.display() + )) + })? { + let path = entry.path(); + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime slot name is not UTF-8", + path.display() + )); + continue; + }; + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!("{}: inspect runtime slot: {error}", path.display())); + continue; + } + }; + if name == CLEANUP_NAMESPACE { + if !file_type.is_dir() { + errors.push(format!( + "{}: runtime cleanup namespace is not a directory", + path.display() + )); + } + continue; + } + if !file_type.is_dir() { + errors.push(format!( + "{}: unexpected non-directory runtime slot entry", + path.display() + )); + continue; + } + match parse_stable_id(&name) { + Ok(id) => { + directories.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + let (tombstones, tombstone_errors) = + scan_cleanup_dirs(cleanup_root, deletion_proof_root).await?; + errors.extend(tombstone_errors); + for instance_id in directories.keys() { + if tombstones.contains_key(instance_id) { + errors.push(format!( + "{instance_id}: runtime slot exists in both active and cleanup namespaces" + )); + } + } + Ok((directories, tombstones, errors)) +} + +async fn scan_cleanup_dirs( + cleanup_root: &Path, + deletion_proof_root: &Path, +) -> Result<(BTreeMap, Vec)> { + let mut tombstones = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "read runtime cleanup namespace {}: {error}", + cleanup_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime cleanup entry under {}: {error}", + cleanup_root.display() + )) + })? { + let path = entry.path(); + if path == deletion_proof_root { + let file_type = entry.file_type().await.map_err(|error| { + recovery_error(format!( + "inspect runtime deletion proof namespace {}: {error}", + path.display() + )) + })?; + if !file_type.is_dir() { + errors.push(format!( + "{}: runtime deletion proof namespace is not a directory", + path.display() + )); + } + continue; + } + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!( + "{}: inspect runtime cleanup entry: {error}", + path.display() + )); + continue; + } + }; + if !file_type.is_dir() { + errors.push(format!( + "{}: unexpected non-directory runtime cleanup entry", + path.display() + )); + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime cleanup name is not UTF-8", + path.display() + )); + continue; + }; + match parse_stable_id(&name) { + Ok(id) => { + tombstones.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + Ok((tombstones, errors)) +} + +struct DeletionProofRepair { + instance_id: Uuid, + proof: PathBuf, + tombstone: PathBuf, + create_tombstone: bool, + restore_journal: bool, +} + +async fn ensure_deletion_proof_root(cleanup_root: &Path, proof_root: &Path) -> Result<()> { + ensure_real_directory(cleanup_root, "runtime cleanup namespace").await?; + ensure_real_directory(proof_root, "runtime deletion proof namespace").await?; + sync_directory(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "sync runtime cleanup namespace {} after ensuring deletion proofs: {error}", + cleanup_root.display() + )) + }) +} + +async fn restore_deletion_proofs(cleanup_root: &Path, proof_root: &Path) -> Result<()> { + let (proofs, mut errors) = scan_deletion_proofs(proof_root).await?; + let mut repairs = Vec::new(); + for (instance_id, proof) in proofs { + let proof_ownership = match read_ownership_file(&proof, instance_id).await { + Ok(ownership) + if matches!( + ownership.phase, + RuntimeSlotPhase::PoolCleanup | RuntimeSlotPhase::LifecycleCleanup { .. } + ) => + { + ownership + } + Ok(ownership) => { + errors.push(format!( + "{instance_id}: deletion proof {} records non-cleanup phase {:?}", + proof.display(), + ownership.phase + )); + continue; + } + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + }; + let tombstone = cleanup_root.join(instance_id.to_string()); + let (create_tombstone, restore_journal) = + match tokio::fs::symlink_metadata(&tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => { + let journal = tombstone.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&journal).await { + Ok(metadata) if metadata.file_type().is_file() => { + match read_ownership_file(&journal, instance_id).await { + Ok(ownership) if ownership == proof_ownership => (false, false), + Ok(_) => { + errors.push(format!( + "{instance_id}: deletion proof {} does not match {}", + proof.display(), + journal.display() + )); + continue; + } + Err(error) => { + errors.push(format!("{instance_id}: {error}")); + continue; + } + } + } + Ok(_) => { + errors.push(format!( + "{instance_id}: cleanup journal {} is not a regular file", + journal.display() + )); + continue; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (false, true), + Err(error) => { + errors.push(format!( + "{instance_id}: inspect cleanup journal {}: {error}", + journal.display() + )); + continue; + } + } + } + Ok(_) => { + errors.push(format!( + "{instance_id}: cleanup tombstone {} is not a real directory", + tombstone.display() + )); + continue; + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => (true, true), + Err(error) => { + errors.push(format!( + "{instance_id}: inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + continue; + } + }; + repairs.push(DeletionProofRepair { + instance_id, + proof, + tombstone, + create_tombstone, + restore_journal, + }); + } + if !errors.is_empty() { + return Err(unresolved_items(errors)); + } + + for repair in repairs { + if repair.create_tombstone { + tokio::fs::create_dir(&repair.tombstone) + .await + .map_err(|error| { + recovery_error(format!( + "{}: recreate cleanup tombstone {} from deletion proof: {error}", + repair.instance_id, + repair.tombstone.display() + )) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + recovery_error(format!( + "{}: sync runtime cleanup namespace after recreating tombstone: {error}", + repair.instance_id + )) + })?; + } + if repair.restore_journal { + let journal = repair.tombstone.join(OWNERSHIP_FILE); + tokio::fs::hard_link(&repair.proof, &journal) + .await + .map_err(|error| { + recovery_error(format!( + "{}: restore cleanup journal {} from deletion proof {}: {error}", + repair.instance_id, + journal.display(), + repair.proof.display() + )) + })?; + sync_directory(&repair.tombstone).await.map_err(|error| { + recovery_error(format!( + "{}: sync restored cleanup tombstone {}: {error}", + repair.instance_id, + repair.tombstone.display() + )) + })?; + } + } + Ok(()) +} + +async fn scan_deletion_proofs(proof_root: &Path) -> Result<(BTreeMap, Vec)> { + let mut proofs = BTreeMap::new(); + let mut errors = Vec::new(); + let mut entries = tokio::fs::read_dir(proof_root).await.map_err(|error| { + recovery_error(format!( + "read runtime deletion proof namespace {}: {error}", + proof_root.display() + )) + })?; + while let Some(entry) = entries.next_entry().await.map_err(|error| { + recovery_error(format!( + "read runtime deletion proof entry under {}: {error}", + proof_root.display() + )) + })? { + let path = entry.path(); + let file_type = match entry.file_type().await { + Ok(file_type) => file_type, + Err(error) => { + errors.push(format!( + "{}: inspect runtime deletion proof: {error}", + path.display() + )); + continue; + } + }; + if !file_type.is_file() { + errors.push(format!( + "{}: runtime deletion proof is not a regular file", + path.display() + )); + continue; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + errors.push(format!( + "{}: runtime deletion proof name is not UTF-8", + path.display() + )); + continue; + }; + match parse_stable_id(&name) { + Ok(id) => { + proofs.insert(id, path); + } + Err(error) => errors.push(format!("{}: {error}", path.display())), + } + } + Ok((proofs, errors)) +} + +async fn ensure_real_directory(path: &Path, label: &str) -> Result<()> { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.file_type().is_dir() => return Ok(()), + Ok(_) => { + return Err(recovery_error(format!( + "{label} {} is not a real directory", + path.display() + ))); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(recovery_error(format!( + "inspect {label} {}: {error}", + path.display() + ))); + } + } + + tokio::fs::create_dir_all(path) + .await + .map_err(|error| recovery_error(format!("create {label} {}: {error}", path.display())))?; + let metadata = tokio::fs::symlink_metadata(path).await.map_err(|error| { + recovery_error(format!( + "inspect created {label} {}: {error}", + path.display() + )) + })?; + if !metadata.file_type().is_dir() { + return Err(recovery_error(format!( + "{label} {} is not a real directory", + path.display() + ))); + } + Ok(()) +} + +async fn rename_to_tombstone( + runtime_root: &Path, + cleanup_root: &Path, + run_dir: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + let tombstone = cleanup_root.join(instance_id.to_string()); + match tokio::fs::symlink_metadata(&tombstone).await { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => { + return Err(format!( + "cleanup tombstone {} already exists", + tombstone.display() + )); + } + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + } + + tokio::fs::rename(run_dir, &tombstone) + .await + .map_err(|error| { + format!( + "move {} to {}: {error}", + run_dir.display(), + tombstone.display() + ) + })?; + sync_directory(runtime_root).await.map_err(|error| { + format!( + "sync runtime slot root {} after rename: {error}", + runtime_root.display() + ) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "sync runtime cleanup namespace {} after rename: {error}", + cleanup_root.display() + ) + })?; + Ok(()) +} + +#[derive(Clone, Copy)] +enum CleanupOwner { + Pool, + Lifecycle(Uuid), +} + +impl CleanupOwner { + fn phase(self) -> RuntimeSlotPhase { + match self { + Self::Pool => RuntimeSlotPhase::PoolCleanup, + Self::Lifecycle(token) => RuntimeSlotPhase::LifecycleCleanup { token }, + } + } +} + +pub(super) async fn tombstone_pool_slot( + runtime_root: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + tombstone_owned_slot(runtime_root, instance_id, CleanupOwner::Pool).await +} + +pub(crate) async fn tombstone_lifecycle_slot( + runtime_root: &Path, + instance_id: Uuid, + owner_token: Uuid, +) -> std::result::Result<(), String> { + tombstone_owned_slot( + runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await +} + +async fn tombstone_owned_slot( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + ensure_real_directory(runtime_root, "runtime slot root") + .await + .map_err(|error| error.to_string())?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace") + .await + .map_err(|error| error.to_string())?; + let run_dir = runtime_root.join(instance_id.to_string()); + let tombstone = cleanup_root.join(instance_id.to_string()); + let run_dir_exists = real_directory_exists(&run_dir, "runtime slot").await?; + let tombstone_exists = real_directory_exists(&tombstone, "cleanup tombstone").await?; + match (run_dir_exists, tombstone_exists) { + (true, false) => { + require_cleanup_owner(&run_dir, instance_id, expected_owner).await?; + rename_to_tombstone(runtime_root, &cleanup_root, &run_dir, instance_id).await + } + (false, true) => { + require_cleanup_owner(&tombstone, instance_id, expected_owner).await?; + Ok(()) + } + (false, false) => match expected_owner { + CleanupOwner::Pool => sync_directory(&cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after prior pool removal: \ + {error}" + ) + }), + CleanupOwner::Lifecycle(_) => Err(format!( + "{instance_id}: runtime slot has neither an active directory nor a cleanup \ + tombstone" + )), + }, + (true, true) => Err(format!( + "{instance_id}: runtime slot exists in both active and cleanup namespaces" + )), + } +} + +async fn require_cleanup_owner( + directory: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result { + let ownership = read_ownership(directory, instance_id).await?; + if ownership.phase != expected_owner.phase() { + return Err(format!( + "{} records {:?} instead of the expected cleanup owner", + directory.join(OWNERSHIP_FILE).display(), + ownership.phase + )); + } + Ok(ownership) +} + +pub(super) async fn remove_pool_tombstone( + runtime_root: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + remove_owned_tombstone(runtime_root, instance_id, CleanupOwner::Pool).await +} + +pub(crate) async fn remove_lifecycle_tombstone( + runtime_root: &Path, + instance_id: Uuid, + owner_token: Uuid, +) -> std::result::Result<(), String> { + remove_owned_tombstone( + runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await +} + +async fn remove_owned_tombstone( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + // Pool maintenance, lifecycle operation locks, and startup isolation each + // serialize removal for one instance ID. The proof protocol relies on that + // per-owner exclusion while it repairs or removes the canonical paths. + let run_dir = runtime_root.join(instance_id.to_string()); + if real_directory_exists(&run_dir, "runtime slot").await? { + return Err(format!( + "{instance_id}: active runtime slot still exists before tombstone removal" + )); + } + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &proof_root) + .await + .map_err(|error| error.to_string())?; + let tombstone = cleanup_root.join(instance_id.to_string()); + let proof = proof_root.join(instance_id.to_string()); + + let proof_exists = match tokio::fs::symlink_metadata(&proof).await { + Ok(metadata) if metadata.file_type().is_file() => true, + Ok(_) => { + return Err(format!( + "runtime deletion proof {} is not a regular file", + proof.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "inspect runtime deletion proof {}: {error}", + proof.display() + )); + } + }; + + if proof_exists { + restore_one_deletion_proof( + &cleanup_root, + &proof_root, + &proof, + &tombstone, + instance_id, + expected_owner, + ) + .await?; + } else if real_directory_exists(&tombstone, "cleanup tombstone").await? { + arm_deletion_proof(&proof_root, &tombstone, &proof, instance_id, expected_owner).await?; + } else { + sync_directory(&cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after prior tombstone removal: \ + {error}" + ) + })?; + return Ok(()); + } + + remove_armed_tombstone(&cleanup_root, &proof_root, &tombstone, &proof, instance_id).await +} + +async fn arm_deletion_proof( + proof_root: &Path, + tombstone: &Path, + proof: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + let ownership = require_cleanup_owner(tombstone, instance_id, expected_owner).await?; + let journal = tombstone.join(OWNERSHIP_FILE); + tokio::fs::hard_link(&journal, proof) + .await + .map_err(|error| { + format!( + "{instance_id}: preserve cleanup ownership from {} in {}: {error}", + journal.display(), + proof.display() + ) + })?; + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {}: {error}", + proof_root.display() + ) + })?; + let proof_ownership = read_ownership_file(proof, instance_id).await?; + if proof_ownership != ownership { + return Err(format!( + "{instance_id}: deletion proof {} changed while it was persisted", + proof.display() + )); + } + Ok(()) +} + +async fn real_directory_exists(path: &Path, label: &str) -> std::result::Result { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.file_type().is_dir() => Ok(true), + Ok(_) => Err(format!( + "{label} {} is not a real directory", + path.display() + )), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(format!("inspect {label} {}: {error}", path.display())), + } +} + +async fn restore_one_deletion_proof( + cleanup_root: &Path, + proof_root: &Path, + proof: &Path, + tombstone: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, +) -> std::result::Result<(), String> { + let proof_ownership = read_ownership_file(proof, instance_id).await?; + if proof_ownership.phase != expected_owner.phase() { + return Err(format!( + "{} records {:?} instead of the expected cleanup owner", + proof.display(), + proof_ownership.phase + )); + } + match tokio::fs::symlink_metadata(tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => {} + Ok(_) => { + return Err(format!( + "cleanup tombstone {} is not a real directory", + tombstone.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::create_dir(tombstone).await.map_err(|error| { + format!( + "{instance_id}: recreate cleanup tombstone {} from deletion proof: {error}", + tombstone.display() + ) + })?; + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime cleanup namespace after recreating tombstone: \ + {error}" + ) + })?; + } + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + } + let journal = tombstone.join(OWNERSHIP_FILE); + match tokio::fs::symlink_metadata(&journal).await { + Ok(metadata) if metadata.file_type().is_file() => { + let ownership = read_ownership_file(&journal, instance_id).await?; + if ownership != proof_ownership { + return Err(format!( + "{instance_id}: deletion proof {} does not match {}", + proof.display(), + journal.display() + )); + } + } + Ok(_) => { + return Err(format!( + "{instance_id}: cleanup journal {} is not a regular file", + journal.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + tokio::fs::hard_link(proof, &journal) + .await + .map_err(|error| { + format!( + "{instance_id}: restore cleanup journal {} from deletion proof {}: {error}", + journal.display(), + proof.display() + ) + })?; + sync_directory(tombstone).await.map_err(|error| { + format!( + "{instance_id}: sync restored cleanup tombstone {}: {error}", + tombstone.display() + ) + })?; + } + Err(error) => { + return Err(format!( + "{instance_id}: inspect cleanup journal {}: {error}", + journal.display() + )); + } + } + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {}: {error}", + proof_root.display() + ) + }) +} + +async fn remove_armed_tombstone( + cleanup_root: &Path, + proof_root: &Path, + tombstone: &Path, + proof: &Path, + instance_id: Uuid, +) -> std::result::Result<(), String> { + let tombstone_exists = match tokio::fs::symlink_metadata(tombstone).await { + Ok(metadata) if metadata.file_type().is_dir() => true, + Ok(_) => { + return Err(format!( + "cleanup tombstone {} is not a real directory", + tombstone.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => false, + Err(error) => { + return Err(format!( + "inspect cleanup tombstone {}: {error}", + tombstone.display() + )); + } + }; + if tombstone_exists { + tokio::fs::remove_dir_all(tombstone) + .await + .map_err(|error| { + format!("remove cleanup tombstone {}: {error}", tombstone.display()) + })?; + } + sync_directory(cleanup_root).await.map_err(|error| { + format!( + "sync runtime cleanup namespace {} after removal: {error}", + cleanup_root.display() + ) + })?; + match tokio::fs::remove_file(proof).await { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(format!( + "{instance_id}: remove runtime deletion proof {}: {error}", + proof.display() + )); + } + } + sync_directory(proof_root).await.map_err(|error| { + format!( + "{instance_id}: sync runtime deletion proof namespace {} after removal: {error}", + proof_root.display() + ) + })?; + Ok(()) +} + +async fn sync_directory(path: &Path) -> std::io::Result<()> { + tokio::fs::File::open(path).await?.sync_all().await +} + +pub(super) async fn finish_ownership_handoff( + run_dir: &Path, + expected_id: Uuid, + expected_backend: BackendKind, + expected_ownership: BackendOwnership, + expected_token: Uuid, +) -> std::result::Result<(), String> { + let mut ownership = read_ownership(run_dir, expected_id).await?; + let path = run_dir.join(OWNERSHIP_FILE); + if ownership.backend != expected_backend { + return Err(format!( + "{} records backend {} instead of {expected_backend}", + path.display(), + ownership.backend + )); + } + if ownership.backend_ownership != expected_ownership { + return Err(format!( + "{} records backend ownership {:?} instead of {:?}", + path.display(), + ownership.backend_ownership, + expected_ownership + )); + } + if !ownership.storage_owned { + return Err(format!( + "{} cannot transfer a slot without owned storage", + path.display() + )); + } + if ownership.phase + == (RuntimeSlotPhase::LifecycleOwned { + token: expected_token, + }) + { + return Ok(()); + } + if ownership.phase + != (RuntimeSlotPhase::Handoff { + token: expected_token, + }) + { + return Err(format!( + "{} does not record expected ownership handoff token {expected_token}", + path.display() + )); + } + ownership.phase = RuntimeSlotPhase::LifecycleOwned { + token: expected_token, + }; + write_ownership(run_dir, &ownership).await +} + +/// Durably transfer a claimed slot into lifecycle cleanup before mutating +/// backend or storage ownership. +pub(crate) async fn begin_lifecycle_cleanup( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + owner_token: Uuid, +) -> std::result::Result<(), String> { + ensure_real_directory(runtime_root, "runtime slot root") + .await + .map_err(|error| error.to_string())?; + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + ensure_real_directory(&cleanup_root, "runtime cleanup namespace") + .await + .map_err(|error| error.to_string())?; + let run_dir = runtime_root.join(instance_id.to_string()); + let tombstone = cleanup_root.join(instance_id.to_string()); + let run_dir_exists = real_directory_exists(&run_dir, "runtime slot").await?; + let tombstone_exists = real_directory_exists(&tombstone, "cleanup tombstone").await?; + let (directory, tombstoned) = match (run_dir_exists, tombstone_exists) { + (true, false) => (run_dir, false), + (false, true) => (tombstone, true), + (false, false) => { + return Err(format!( + "{instance_id}: lifecycle-owned runtime slot has neither an active directory nor \ + a cleanup tombstone" + )); + } + (true, true) => { + return Err(format!( + "{instance_id}: lifecycle-owned runtime slot exists in both active and cleanup \ + namespaces" + )); + } + }; + let mut ownership = read_ownership(&directory, instance_id).await?; + if ownership.backend != backend { + return Err(format!( + "{} records backend {} instead of lifecycle backend {backend}", + directory.join(OWNERSHIP_FILE).display(), + ownership.backend + )); + } + let current_phase = ownership.phase; + match current_phase { + RuntimeSlotPhase::Handoff { token } + | RuntimeSlotPhase::LifecycleOwned { token } + | RuntimeSlotPhase::LifecycleCleanup { token } + if token == owner_token => {} + RuntimeSlotPhase::Handoff { token } + | RuntimeSlotPhase::LifecycleOwned { token } + | RuntimeSlotPhase::LifecycleCleanup { token } => { + return Err(format!( + "{} records lifecycle token {token} instead of {owner_token}", + directory.join(OWNERSHIP_FILE).display() + )); + } + phase => { + return Err(format!( + "{} records pool phase {phase:?} instead of lifecycle ownership", + directory.join(OWNERSHIP_FILE).display() + )); + } + } + if matches!( + current_phase, + RuntimeSlotPhase::Handoff { .. } | RuntimeSlotPhase::LifecycleOwned { .. } + ) && !ownership.storage_owned + { + return Err(format!( + "{} records lifecycle ownership without owned storage", + directory.join(OWNERSHIP_FILE).display() + )); + } + if tombstoned { + if ownership.phase != (RuntimeSlotPhase::LifecycleCleanup { token: owner_token }) { + return Err(format!( + "{} was tombstoned before lifecycle cleanup was committed", + directory.join(OWNERSHIP_FILE).display() + )); + } + return Ok(()); + } + if ownership.phase == (RuntimeSlotPhase::LifecycleCleanup { token: owner_token }) { + return Ok(()); + } + ownership.phase = RuntimeSlotPhase::LifecycleCleanup { token: owner_token }; + write_ownership(&directory, &ownership).await +} + +pub(super) async fn read_ownership( + run_dir: &Path, + expected_id: Uuid, +) -> std::result::Result { + let path = run_dir.join(OWNERSHIP_FILE); + read_ownership_file(&path, expected_id).await +} + +async fn read_ownership_file( + path: &Path, + expected_id: Uuid, +) -> std::result::Result { + let metadata = tokio::fs::symlink_metadata(&path) + .await + .map_err(|error| format!("inspect {}: {error}", path.display()))?; + if !metadata.file_type().is_file() { + return Err(format!("{} is not a regular file", path.display())); + } + let encoded = tokio::fs::read(&path) + .await + .map_err(|error| format!("read {}: {error}", path.display()))?; + let ownership: RuntimeSlotOwnership = serde_json::from_slice(&encoded) + .map_err(|error| format!("decode {}: {error}", path.display()))?; + validate_ownership(path, &ownership, expected_id)?; + Ok(ownership) +} + +fn validate_ownership( + path: &Path, + ownership: &RuntimeSlotOwnership, + expected_id: Uuid, +) -> std::result::Result<(), String> { + if ownership.version != OWNERSHIP_VERSION { + return Err(format!( + "{} has unsupported ownership version {}", + path.display(), + ownership.version + )); + } + if ownership.instance_id != expected_id { + return Err(format!( + "{} records instance {} instead of directory {expected_id}", + path.display(), + ownership.instance_id + )); + } + Ok(()) +} + +pub(super) async fn write_ownership( + run_dir: &Path, + ownership: &RuntimeSlotOwnership, +) -> std::result::Result<(), String> { + let encoded = serde_json::to_vec(ownership) + .map_err(|error| format!("encode ownership record: {error}"))?; + let path = run_dir.join(OWNERSHIP_FILE); + let temporary = run_dir.join(format!(".ownership-{}.tmp", Uuid::new_v4())); + let result = async { + let mut file = tokio::fs::OpenOptions::new() + .create_new(true) + .write(true) + .open(&temporary) + .await?; + file.write_all(&encoded).await?; + file.sync_all().await?; + drop(file); + tokio::fs::rename(&temporary, &path).await?; + tokio::fs::File::open(run_dir).await?.sync_all().await + } + .await; + if let Err(error) = result { + let _ = tokio::fs::remove_file(&temporary).await; + return Err(format!("persist {}: {error}", path.display())); + } + Ok(()) +} + +fn parse_stable_id(value: &str) -> std::result::Result { + let id = Uuid::parse_str(value).map_err(|error| format!("slot ID is not a UUID: {error}"))?; + if value != id.to_string() { + return Err(format!("slot ID must use canonical UUID form {id}")); + } + Ok(id) +} + +fn recovery_error(message: String) -> BlazeDaemonError { + BlazeDaemonError::RecoveryRequired(message) +} + +fn unresolved_items(errors: Vec) -> BlazeDaemonError { + recovery_error(format!( + "runtime slot reconciliation found {} unresolved item(s): {}", + errors.len(), + errors.join("; ") + )) +} + +#[cfg(test)] +mod tests { + use std::future; + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use blaze_core::backend::SpawnRequest; + use blaze_core::lifecycle::StartPath; + use blaze_core::storage::{AcquireOpts, PoolStatus, StorageAcquireError, StorageSlot}; + use blaze_core::{BlazeError, Result as CoreResult}; + + use super::*; + use crate::spawner::{BackendSpawner, DynBackendInstance, DynSpawner, SpawnFailure}; + + #[derive(Default)] + struct RecordingStorage { + owned: Mutex>, + fail_once: Mutex>, + events: Arc>>, + supports_recovery: bool, + inventory_error: Option, + inventory_override: Option>, + pending_inventory: bool, + pending_release: bool, + } + + impl RecordingStorage { + fn with_ids(ids: impl IntoIterator, events: Arc>>) -> Self { + Self { + owned: Mutex::new(ids.into_iter().collect()), + fail_once: Mutex::new(BTreeSet::new()), + events, + supports_recovery: true, + inventory_error: None, + inventory_override: None, + pending_inventory: false, + pending_release: false, + } + } + + fn fail_next_release(&self, instance_id: Uuid) { + self.fail_once + .lock() + .expect("release failures") + .insert(instance_id); + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> CoreResult { + Ok(true) + } + + async fn acquire( + &self, + _opts: &AcquireOpts, + ) -> std::result::Result { + Err(StorageAcquireError::clean(BlazeError::StorageError { + msg: "test storage does not acquire slots".into(), + })) + } + + async fn release(&self, slot: StorageSlot) -> CoreResult<()> { + self.release_by_id(&slot.id).await + } + + async fn release_by_id(&self, instance_id: &str) -> CoreResult<()> { + let id = Uuid::parse_str(instance_id).map_err(|error| BlazeError::StorageError { + msg: format!("invalid test slot id {instance_id}: {error}"), + })?; + self.events + .lock() + .expect("events") + .push(format!("storage:{id}")); + if self.pending_release { + future::pending::<()>().await; + } + if self.fail_once.lock().expect("release failures").remove(&id) { + return Err(BlazeError::StorageError { + msg: format!("injected release failure for {id}"), + }); + } + self.owned.lock().expect("owned slots").remove(&id); + Ok(()) + } + + async fn reconstruct(&self, instance_id: &str) -> CoreResult { + Err(BlazeError::StorageError { + msg: format!("test storage cannot reconstruct {instance_id}"), + }) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> CoreResult<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> CoreResult { + Ok(0) + } + + fn supports_runtime_pool_recovery(&self) -> bool { + self.supports_recovery + } + + async fn list_owned_ids(&self) -> CoreResult> { + if self.pending_inventory { + future::pending::<()>().await; + } + if let Some(message) = &self.inventory_error { + return Err(BlazeError::StorageError { + msg: message.clone(), + }); + } + if let Some(ids) = &self.inventory_override { + return Ok(ids.clone()); + } + Ok(self + .owned + .lock() + .expect("owned slots") + .iter() + .map(Uuid::to_string) + .collect()) + } + } + + struct RecordingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for RecordingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + Ok(()) + } + } + + struct FailingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for FailingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + Err(BlazeError::BackendError { + msg: format!("injected backend cleanup failure for {instance_id}"), + }) + } + } + + struct PendingSpawner { + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for PendingSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + future::pending::>().await + } + } + + struct DelayedSpawner { + delay: Duration, + events: Arc>>, + } + + #[async_trait] + impl BackendSpawner for DelayedSpawner { + async fn spawn( + &self, + _request: SpawnRequest, + ) -> std::result::Result { + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "test spawner does not spawn instances".into(), + })) + } + + async fn probe(&self, _binary_path: &Path) -> CoreResult { + Ok(true) + } + + async fn cleanup_orphan(&self, instance_id: Uuid, _run_dir: &Path) -> CoreResult<()> { + self.events + .lock() + .expect("events") + .push(format!("backend:{instance_id}")); + tokio::time::sleep(self.delay).await; + Ok(()) + } + } + + fn registry(kind: BackendKind, spawner: DynSpawner) -> SpawnerRegistry { + let mut registry = SpawnerRegistry::new(); + registry.insert(kind, spawner); + registry + } + + fn write_ownership( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + backend_ownership: BackendOwnership, + storage_owned: bool, + ) -> PathBuf { + write_ownership_with_phase( + runtime_root, + instance_id, + backend, + backend_ownership, + storage_owned, + RuntimeSlotPhase::Building, + ) + } + + fn write_ownership_with_phase( + runtime_root: &Path, + instance_id: Uuid, + backend: BackendKind, + backend_ownership: BackendOwnership, + storage_owned: bool, + phase: RuntimeSlotPhase, + ) -> PathBuf { + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend, + backend_ownership, + storage_owned, + phase, + }; + std::fs::write( + run_dir.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("serialize ownership"), + ) + .expect("write ownership"); + run_dir + } + + fn write_tombstone(runtime_root: &Path, instance_id: Uuid) -> PathBuf { + let tombstone = runtime_root + .join(CLEANUP_NAMESPACE) + .join(instance_id.to_string()); + std::fs::create_dir_all(&tombstone).expect("tombstone"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + storage_owned: false, + phase: RuntimeSlotPhase::PoolCleanup, + }; + std::fs::write( + tombstone.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("serialize tombstone ownership"), + ) + .expect("write tombstone ownership"); + tombstone + } + + async fn arm_test_deletion_proof( + runtime_root: &Path, + instance_id: Uuid, + expected_owner: CleanupOwner, + ) -> PathBuf { + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + ensure_deletion_proof_root(&cleanup_root, &proof_root) + .await + .expect("proof root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + let proof = proof_root.join(instance_id.to_string()); + arm_deletion_proof(&proof_root, &tombstone, &proof, instance_id, expected_owner) + .await + .expect("arm deletion proof"); + proof + } + + #[tokio::test] + async fn reconcile_cleans_backend_before_storage_and_run_dir() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let cleaned = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect("reconcile"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_retains_storage_after_backend_cleanup_failure() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(FailingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("backend failure must retain later owners"); + + assert!( + error + .to_string() + .contains("injected backend cleanup failure") + ); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{instance_id}")] + ); + assert!(run_dir.exists()); + assert!( + storage + .owned + .lock() + .expect("owned slots") + .contains(&instance_id) + ); + } + + #[tokio::test] + async fn reconcile_releases_provider_only_slot_by_id() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("reconcile"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + } + + #[tokio::test] + async fn reconcile_protects_durable_lifecycle_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + RuntimeSlotPhase::LifecycleOwned { token: owner_token }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("durable owner must be excluded"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_checks_live_warm_owner_missing_from_inventory() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: None, + clean_terminal: false, + }; + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("live warm owner requires its canonical run dir"); + + let message = error.to_string(); + assert!(message.contains(&instance_id.to_string())); + assert!(message.contains("canonical runtime directory")); + assert!(message.contains("is missing")); + } + + #[tokio::test] + async fn reconcile_rejects_warm_owner_without_run_dir() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::NotStarted, + state: SandboxState::Creating, + operation: Some(OperationKind::Create), + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("durable warm owner must retain its canonical run dir"); + + assert!(error.to_string().contains("missing canonical")); + } + + #[test] + fn runtime_location_survives_warm_reclassification_and_reload() { + let temp = tempfile::tempdir().expect("tempdir"); + let mut sandbox = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:sandbox-location".into(), + StartPath::Cold, + "sandbox-location".into(), + ); + sandbox + .transition(SandboxState::Creating) + .expect("creating"); + sandbox.transition(SandboxState::Running).expect("running"); + sandbox.transition(SandboxState::Reset).expect("reset"); + sandbox.transition(SandboxState::Warm).expect("warm"); + sandbox + .transition(SandboxState::Creating) + .expect("warm claim"); + sandbox.persist(temp.path()).expect("persist sandbox"); + let sandbox = SandboxInstance::load(temp.path(), sandbox.id).expect("reload sandbox"); + assert_eq!(sandbox.start_path, StartPath::Warm); + assert_eq!( + runtime_dir(temp.path(), sandbox.runtime_location, sandbox.id), + temp.path().join(sandbox.id.to_string()) + ); + + let mut warm_pool = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:warm-location".into(), + StartPath::Warm, + "warm-location".into(), + ); + warm_pool.runtime_location = RuntimeLocation::WarmPool; + warm_pool.persist(temp.path()).expect("persist warm owner"); + let warm_pool = + SandboxInstance::load(temp.path(), warm_pool.id).expect("reload warm owner"); + assert_eq!( + runtime_dir(temp.path(), warm_pool.runtime_location, warm_pool.id), + temp.path() + .join("runtime-pool") + .join(warm_pool.id.to_string()) + ); + } + + #[tokio::test] + async fn reconcile_rejects_cold_lifecycle_run_dir_collision() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::Sandbox, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: None, + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("cold lifecycle must not adopt a pool run dir"); + + assert!(error.to_string().contains("sandbox runtime owner")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_transferred_backend_mismatch() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Runc, + backend_ownership: BackendOwnership::Running, + state: SandboxState::Running, + operation: None, + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("backend mismatch must stop startup"); + + let message = error.to_string(); + assert!(message.contains("records backend mock")); + assert!(message.contains("durable lifecycle owns runc")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_does_not_protect_destroyed_lifecycle_record() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::Sandbox, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: None, + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("destroyed lifecycle record does not own runtime resources"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + } + + #[tokio::test] + async fn reconcile_protects_nonterminal_destroyed_owners() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let live_id = Uuid::new_v4(); + let journal_id = Uuid::new_v4(); + let mut live = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:live-destroyed".into(), + StartPath::Cold, + "live-destroyed".into(), + ); + live.id = live_id; + live.transition(SandboxState::Destroyed).expect("destroyed"); + live.backend_ownership = BackendOwnership::Running; + let mut journal = SandboxInstance::new( + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:journal-destroyed".into(), + StartPath::Cold, + "journal-destroyed".into(), + ); + journal.id = journal_id; + journal + .begin_operation(blaze_core::lifecycle::OperationKind::Destroy) + .expect("destroy journal"); + journal + .transition(SandboxState::Destroyed) + .expect("destroyed"); + journal.backend_ownership = BackendOwnership::Stopped; + let owners = HashMap::from([ + (live_id, DurableRuntimeOwner::from(&live)), + (journal_id, DurableRuntimeOwner::from(&journal)), + ]); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([live_id, journal_id], events.clone()); + + let cleaned = + reconcile_runtime_slots(&runtime_root, &owners, &storage, &SpawnerRegistry::new()) + .await + .expect("nonterminal lifecycle records retain cleanup ownership"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + } + + #[tokio::test] + async fn reconcile_does_not_call_provider_without_recovery_contract() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage { + owned: Mutex::new(BTreeSet::from([instance_id])), + fail_once: Mutex::new(BTreeSet::new()), + events: events.clone(), + supports_recovery: false, + ..RecordingStorage::default() + }; + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("unsupported provider must stop before cleanup"); + + assert!( + error + .to_string() + .contains("cannot inventory slots for runtime recovery") + ); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_namespace_alias_before_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = + Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let alias = runtime_root.join(instance_id.to_string().to_uppercase()); + std::fs::create_dir_all(&alias).expect("alias run dir"); + let ownership = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Running, + storage_owned: true, + phase: RuntimeSlotPhase::Building, + }; + std::fs::write( + alias.join(OWNERSHIP_FILE), + serde_json::to_vec(&ownership).expect("ownership"), + ) + .expect("alias ownership"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("noncanonical alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(alias.exists()); + assert!( + storage + .owned + .lock() + .expect("owned slots") + .contains(&instance_id) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconcile_rejects_uuid_symlink_before_cleanup() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let instance_id = Uuid::new_v4(); + let target = tempfile::tempdir().expect("target"); + let alias = runtime_root.join(instance_id.to_string()); + symlink(target.path(), &alias).expect("runtime symlink"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("runtime symlink must stop preflight"); + + assert!(error.to_string().contains("non-directory runtime slot")); + assert!(events.lock().expect("events").is_empty()); + assert!( + std::fs::symlink_metadata(&alias) + .expect("alias") + .is_symlink() + ); + assert!(target.path().is_dir()); + } + + #[cfg(unix)] + #[tokio::test] + async fn reconcile_rejects_symlinked_runtime_roots() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_target = tempfile::tempdir().expect("runtime target"); + let runtime_root = temp.path().join("runtime-pool"); + symlink(runtime_target.path(), &runtime_root).expect("runtime root symlink"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let runtime_error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("runtime root symlink must be rejected"); + + assert!(runtime_error.to_string().contains("not a real directory")); + assert!(events.lock().expect("events").is_empty()); + + let real_runtime_root = temp.path().join("real-runtime-pool"); + std::fs::create_dir_all(&real_runtime_root).expect("real runtime root"); + let cleanup_target = tempfile::tempdir().expect("cleanup target"); + symlink( + cleanup_target.path(), + real_runtime_root.join(CLEANUP_NAMESPACE), + ) + .expect("cleanup root symlink"); + + let cleanup_error = reconcile_runtime_slots( + &real_runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("cleanup root symlink must be rejected"); + + assert!(cleanup_error.to_string().contains("not a real directory")); + assert!(events.lock().expect("events").is_empty()); + assert!(cleanup_target.path().is_dir()); + } + + #[tokio::test] + async fn reconcile_rejects_inventory_error_before_backend_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.inventory_error = Some("injected inventory failure".to_string()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("inventory failure must stop preflight"); + + assert!(error.to_string().contains("injected inventory failure")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_noncanonical_provider_id_before_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = + Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.inventory_override = Some(vec![instance_id.to_string().to_uppercase()]); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let error = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("provider alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_resumes_tombstone_without_external_cleanup() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + let cleaned = reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect("resume tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_resumes_pool_tombstone_after_journal_unlink() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let nested = tombstone.join("partially-removed"); + std::fs::create_dir_all(&nested).expect("partial directory"); + std::fs::write(nested.join("leftover"), b"data").expect("partial file"); + let proof = arm_test_deletion_proof(&runtime_root, instance_id, CleanupOwner::Pool).await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("resume partial tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + assert_eq!( + reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new() + ) + .await + .expect("idempotent reconciliation"), + 0 + ); + } + + #[tokio::test] + async fn reconcile_resumes_lifecycle_tombstone_after_journal_unlink() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("resume partial lifecycle tombstone"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + } + + #[tokio::test] + async fn reconcile_finishes_deletion_when_only_the_proof_remains() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let proof = arm_test_deletion_proof(&runtime_root, instance_id, CleanupOwner::Pool).await; + std::fs::remove_dir_all(&tombstone).expect("simulate completed recursive deletion"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("finish deletion from proof"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + assert!(!proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_partial_tombstone_without_deletion_proof() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("missing deletion proof must stop recovery"); + + assert!(error.to_string().contains("ownership.json")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_mismatched_deletion_proof() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let other_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let other_tombstone = write_tombstone(&runtime_root, other_id); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let proof_root = cleanup_root.join(DELETION_PROOF_NAMESPACE); + std::fs::create_dir_all(&proof_root).expect("proof root"); + let proof = proof_root.join(instance_id.to_string()); + std::fs::hard_link(other_tombstone.join(OWNERSHIP_FILE), &proof).expect("mismatched proof"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("mismatched deletion proof must stop recovery"); + + assert!(error.to_string().contains("records instance")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_deletion_proof_that_differs_from_its_journal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let proof_root = runtime_root + .join(CLEANUP_NAMESPACE) + .join(DELETION_PROOF_NAMESPACE); + std::fs::create_dir_all(&proof_root).expect("proof root"); + let proof = proof_root.join(instance_id.to_string()); + let mismatched = RuntimeSlotOwnership { + version: OWNERSHIP_VERSION, + instance_id, + backend: BackendKind::Runc, + backend_ownership: BackendOwnership::Stopped, + storage_owned: false, + phase: RuntimeSlotPhase::PoolCleanup, + }; + std::fs::write( + &proof, + serde_json::to_vec(&mismatched).expect("proof ownership"), + ) + .expect("mismatched proof"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("proof and journal mismatch must stop recovery"); + + assert!(error.to_string().contains("does not match")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_partial_lifecycle_tombstone_with_wrong_token() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let proof_token = Uuid::new_v4(); + let durable_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: proof_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(proof_token), + ) + .await; + std::fs::remove_file(tombstone.join(OWNERSHIP_FILE)).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(durable_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("wrong lifecycle proof token must stop recovery"); + + assert!(error.to_string().contains("cleanup token")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + } + + #[tokio::test] + async fn reconcile_restores_but_defers_nonterminal_lifecycle_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let proof = arm_test_deletion_proof( + &runtime_root, + instance_id, + CleanupOwner::Lifecycle(owner_token), + ) + .await; + let journal = tombstone.join(OWNERSHIP_FILE); + std::fs::remove_file(&journal).expect("unlink journal"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: Some(OperationKind::Destroy), + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("nonterminal lifecycle cleanup remains lifecycle-owned"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + assert!(proof.exists()); + assert!(journal.is_file()); + } + + #[cfg(unix)] + #[tokio::test] + async fn tombstone_removal_rejects_a_linked_cleanup_root() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + let external = tempfile::tempdir().expect("external"); + let sentinel = external.path().join("keep"); + std::fs::write(&sentinel, b"keep").expect("sentinel"); + symlink(external.path(), runtime_root.join(CLEANUP_NAMESPACE)).expect("cleanup link"); + + let error = remove_pool_tombstone(&runtime_root, Uuid::new_v4()) + .await + .expect_err("linked cleanup root must be rejected"); + + assert!(error.contains("not a real directory")); + assert_eq!(std::fs::read(&sentinel).expect("sentinel remains"), b"keep"); + assert!(!external.path().join(DELETION_PROOF_NAMESPACE).exists()); + } + + #[tokio::test] + async fn reconcile_resumes_tombstone_for_clean_terminal_owner() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let active = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(active, &tombstone).expect("tombstone lifecycle slot"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: None, + runtime_owner_token: Some(owner_token), + clean_terminal: true, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("clean terminal owner no longer owns external resources"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_tombstone_alias_before_removal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let valid_id = Uuid::new_v4(); + let valid = write_tombstone(&runtime_root, valid_id); + let alias_id = Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").expect("fixed UUID"); + let alias = cleanup_root.join(alias_id.to_string().to_uppercase()); + std::fs::create_dir_all(&alias).expect("alias tombstone"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("tombstone alias must stop preflight"); + + assert!(error.to_string().contains("canonical UUID")); + assert!(events.lock().expect("events").is_empty()); + assert!(valid.exists()); + assert!(alias.exists()); + } + + #[tokio::test] + async fn reconcile_preserves_tombstone_that_conflicts_with_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("storage and tombstone conflict must stop preflight"); + + assert!(error.to_string().contains("tombstone conflicts")); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test] + async fn reconcile_preserves_tombstone_owned_by_lifecycle() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let tombstone = write_tombstone(&runtime_root, instance_id); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::RecoveryRequired, + operation: None, + runtime_owner_token: Some(Uuid::new_v4()), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("lifecycle and tombstone conflict must stop preflight"); + + assert!( + error + .to_string() + .contains("collides with durable lifecycle") + ); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_bounds_pending_inventory() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([], events.clone()); + storage.pending_inventory = true; + let started = tokio::time::Instant::now(); + + let error = reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("pending inventory must time out"); + + assert!(error.to_string().contains("shared startup deadline")); + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert!(events.lock().expect("events").is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_retains_storage_while_backend_is_pending() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(PendingSpawner { + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("pending backend must time out"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{instance_id}")] + ); + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("ownership retained"); + assert_eq!(ownership.backend_ownership, BackendOwnership::Running); + assert!(ownership.storage_owned); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_deadline_persists_backend_phase_before_pending_storage() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let mut storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.pending_release = true; + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("pending storage must time out"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + let ownership = read_ownership(&run_dir, instance_id) + .await + .expect("ownership retained"); + assert_eq!(ownership.backend_ownership, BackendOwnership::Stopped); + assert!(ownership.storage_owned); + } + + #[tokio::test(start_paused = true)] + async fn reconcile_uses_one_deadline_for_all_candidates() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let first = Uuid::parse_str("11111111-1111-4111-8111-111111111111").expect("first UUID"); + let second = Uuid::parse_str("22222222-2222-4222-8222-222222222222").expect("second UUID"); + let first_dir = write_ownership( + &runtime_root, + first, + BackendKind::Mock, + BackendOwnership::Running, + false, + ); + let second_dir = write_ownership( + &runtime_root, + second, + BackendKind::Mock, + BackendOwnership::Running, + false, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(DelayedSpawner { + delay: Duration::from_secs(6), + events: events.clone(), + }), + ); + let started = tokio::time::Instant::now(); + + reconcile_runtime_slots_until( + started + Duration::from_secs(10), + &runtime_root, + &HashMap::new(), + &storage, + &spawners, + ) + .await + .expect_err("the second cleanup must share the first deadline"); + + assert_eq!(started.elapsed(), Duration::from_secs(10)); + assert_eq!( + *events.lock().expect("events"), + vec![format!("backend:{first}"), format!("backend:{second}")] + ); + assert!(!first_dir.exists()); + let second_ownership = read_ownership(&second_dir, second) + .await + .expect("second ownership retained"); + assert_eq!( + second_ownership.backend_ownership, + BackendOwnership::Running + ); + } + + #[tokio::test] + async fn reconcile_rejects_run_dir_without_ownership() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("missing ownership must stop startup"); + + assert!(error.to_string().contains(&instance_id.to_string())); + assert!(error.to_string().contains(OWNERSHIP_FILE)); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_removes_empty_slot_left_before_first_journal() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = runtime_root.join(instance_id.to_string()); + std::fs::create_dir_all(&run_dir).expect("run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("empty pre-journal slot has no external owner"); + + assert_eq!(cleaned, 1); + assert!(events.lock().expect("events").is_empty()); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_reports_corrupt_and_unregistered_owners() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let corrupt_id = Uuid::new_v4(); + let corrupt_dir = runtime_root.join(corrupt_id.to_string()); + std::fs::create_dir_all(&corrupt_dir).expect("corrupt run dir"); + std::fs::write(corrupt_dir.join(OWNERSHIP_FILE), b"{not-json").expect("corrupt ownership"); + let unregistered_id = Uuid::new_v4(); + write_ownership( + &runtime_root, + unregistered_id, + BackendKind::Runc, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([corrupt_id, unregistered_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("ambiguous owners must stop startup"); + let message = error.to_string(); + + assert!(message.contains(&corrupt_id.to_string())); + assert!(message.contains("decode")); + assert!(message.contains(&unregistered_id.to_string())); + assert!(message.contains("no recovery spawner")); + assert!(events.lock().expect("events").is_empty()); + assert!(corrupt_dir.exists()); + } + + #[tokio::test] + async fn reconcile_persists_backend_cleanup_before_storage_retry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Running, + true, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + storage.fail_next_release(instance_id); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("storage failure must retain the journal"); + + assert_eq!( + *events.lock().expect("events"), + vec![ + format!("backend:{instance_id}"), + format!("storage:{instance_id}") + ] + ); + let persisted = read_ownership(&run_dir, instance_id) + .await + .expect("persisted cleanup phase"); + assert_eq!(persisted.phase, RuntimeSlotPhase::PoolCleanup); + assert_eq!(persisted.backend_ownership, BackendOwnership::Stopped); + assert!(persisted.storage_owned); + + events.lock().expect("events").clear(); + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("stopped backend needs no recovery spawner"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_continues_safe_cleanup_before_returning_error() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let good_id = Uuid::new_v4(); + let bad_id = Uuid::new_v4(); + let good_dir = write_ownership( + &runtime_root, + good_id, + BackendKind::Mock, + BackendOwnership::Stopped, + true, + ); + std::fs::create_dir_all(runtime_root.join(bad_id.to_string())).expect("bad run dir"); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([good_id, bad_id], events.clone()); + let spawners = registry( + BackendKind::Mock, + Arc::new(RecordingSpawner { + events: events.clone(), + }), + ); + + reconcile_runtime_slots(&runtime_root, &HashMap::new(), &storage, &spawners) + .await + .expect_err("one ambiguous slot must stop startup"); + + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{good_id}")] + ); + assert!(!good_dir.exists()); + assert!(runtime_root.join(bad_id.to_string()).exists()); + } + + #[tokio::test] + async fn reconcile_rejects_unknown_runtime_root_entry() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + std::fs::create_dir_all(&runtime_root).expect("runtime root"); + std::fs::write(runtime_root.join("unexpected"), b"not a slot").expect("entry"); + let storage = RecordingStorage::default(); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("unknown entry must stop startup"); + + assert!(error.to_string().contains("unexpected non-directory")); + } + + #[tokio::test] + async fn handoff_and_cleanup_transitions_are_idempotent_and_token_bound() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::Handoff { token: owner_token }, + ); + + for _ in 0..2 { + finish_ownership_handoff( + &run_dir, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + owner_token, + ) + .await + .expect("idempotent lifecycle handoff"); + } + let wrong_token = Uuid::new_v4(); + begin_lifecycle_cleanup(&runtime_root, instance_id, BackendKind::Mock, wrong_token) + .await + .expect_err("wrong cleanup token"); + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("ownership after rejected token") + .phase, + RuntimeSlotPhase::LifecycleOwned { token: owner_token } + ); + + for _ in 0..2 { + begin_lifecycle_cleanup(&runtime_root, instance_id, BackendKind::Mock, owner_token) + .await + .expect("idempotent lifecycle cleanup"); + } + assert_eq!( + read_ownership(&run_dir, instance_id) + .await + .expect("cleanup ownership") + .phase, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token } + ); + } + + #[tokio::test] + async fn reconcile_cleans_an_uncommitted_handoff_as_pool_owned() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::Handoff { + token: Uuid::new_v4(), + }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("uncommitted handoff remains pool-owned"); + + assert_eq!(cleaned, 1); + assert_eq!( + *events.lock().expect("events"), + vec![format!("storage:{instance_id}")] + ); + assert!(!run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_rejects_lifecycle_marker_without_lifecycle_state() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let instance_id = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::NotStarted, + true, + RuntimeSlotPhase::LifecycleOwned { + token: Uuid::new_v4(), + }, + ); + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([instance_id], events.clone()); + + let error = reconcile_runtime_slots( + &runtime_root, + &HashMap::new(), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect_err("lifecycle marker without state is ambiguous"); + + assert!(error.to_string().contains("lifecycle ownership")); + assert!(events.lock().expect("events").is_empty()); + assert!(run_dir.exists()); + } + + #[tokio::test] + async fn reconcile_defers_a_valid_lifecycle_cleanup_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_root = temp.path().join("runtime-pool"); + let cleanup_root = runtime_root.join(CLEANUP_NAMESPACE); + let instance_id = Uuid::new_v4(); + let owner_token = Uuid::new_v4(); + let run_dir = write_ownership_with_phase( + &runtime_root, + instance_id, + BackendKind::Mock, + BackendOwnership::Stopped, + false, + RuntimeSlotPhase::LifecycleCleanup { token: owner_token }, + ); + std::fs::create_dir_all(&cleanup_root).expect("cleanup root"); + let tombstone = cleanup_root.join(instance_id.to_string()); + std::fs::rename(run_dir, &tombstone).expect("tombstone runtime"); + let owner = DurableRuntimeOwner { + instance_id, + runtime_location: RuntimeLocation::WarmPool, + backend: BackendKind::Mock, + backend_ownership: BackendOwnership::Stopped, + state: SandboxState::Destroyed, + operation: Some(OperationKind::Destroy), + runtime_owner_token: Some(owner_token), + clean_terminal: false, + }; + let events = Arc::new(Mutex::new(Vec::new())); + let storage = RecordingStorage::with_ids([], events.clone()); + + let cleaned = reconcile_runtime_slots( + &runtime_root, + &HashMap::from([(instance_id, owner)]), + &storage, + &SpawnerRegistry::new(), + ) + .await + .expect("lifecycle must finish its own cleanup"); + + assert_eq!(cleaned, 0); + assert!(events.lock().expect("events").is_empty()); + assert!(tombstone.exists()); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index f92384560a..c77662dec5 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -2,16 +2,16 @@ //! Recoverable sandbox create, warm activation, destroy, and startup cleanup. use std::collections::{BTreeSet, HashMap}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, Weak}; use std::time::Duration; use blaze_core::BlazeError; use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::lifecycle::{ - BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, + BackendOwnership, OperationKind, RuntimeLocation, SandboxInstance, SandboxState, StartPath, }; -use blaze_core::policy::RuntimeDecision; +use blaze_core::policy::{RuntimeDecision, parse_duration}; use blaze_core::pool::{PoolKey, PoolManager}; use blaze_core::storage::{AcquireOpts, StorageProvider, StorageSlot}; use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; @@ -21,6 +21,13 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::guest::{GuestClient, GuestExecResult, MAX_GUEST_FILE_BYTES}; use crate::metrics::Metrics; +#[cfg(test)] +use crate::runtime_pool::RuntimePoolStatus; +use crate::runtime_pool::{ + DurableRuntimeOwner, PoolPrototype, RuntimePoolLease, RuntimeWarmPool, begin_lifecycle_cleanup, + reconcile_runtime_slots, remove_lifecycle_tombstone, runtime_dir as derive_runtime_dir, + tombstone_lifecycle_slot, +}; use crate::spawner::{DynBackendInstance, SpawnerRegistry}; const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); @@ -78,6 +85,7 @@ pub struct SandboxManager { backend_instances: Arc>>, operation_locks: Mutex>>>, pool: Arc>, + runtime_pool: Arc, spawners: Arc, active_backend: BackendKind, storage: Arc, @@ -98,6 +106,10 @@ pub struct SandboxManagerInit { pub state_dir: PathBuf, pub rootfs_size: u64, pub mem_size: u64, + pub pool_size: usize, + pub prefork: bool, + pub default_warm_ttl: String, + pub gc_interval: String, } /// Shared read-only resources retained by [`crate::state::ServerState`]. @@ -110,7 +122,7 @@ pub struct SandboxManagerResources { impl SandboxManager { /// Build a manager around state loaded from the durable state directory. - pub fn new(init: SandboxManagerInit) -> (Self, SandboxManagerResources) { + pub fn new(init: SandboxManagerInit) -> Result<(Self, SandboxManagerResources)> { let SandboxManagerInit { instances, pool, @@ -120,34 +132,63 @@ impl SandboxManager { state_dir, rootfs_size, mem_size, + pool_size, + prefork, + default_warm_ttl, + gc_interval, } = init; let instances = Arc::new(Mutex::new(instances)); let backend_instances = Arc::new(Mutex::new(HashMap::new())); let pool = Arc::new(Mutex::new(pool)); let metrics = Arc::new(Metrics::new()); + let spawners = Arc::new(spawners); + let cancellation = CancellationToken::new(); + let default_warm_ttl = parse_duration(&default_warm_ttl).ok_or_else(|| { + BlazeDaemonError::Internal( + "pool.default_warm_ttl passed validation but could not be parsed".to_string(), + ) + })?; + let gc_interval = parse_duration(&gc_interval).ok_or_else(|| { + BlazeDaemonError::Internal( + "pool.gc_interval passed validation but could not be parsed".to_string(), + ) + })?; + let runtime_pool = RuntimeWarmPool::new( + pool_size, + prefork, + rootfs_size, + mem_size, + state_dir.join("runtime-pool"), + storage.clone(), + spawners.clone(), + default_warm_ttl, + gc_interval, + cancellation.clone(), + )?; let resources = SandboxManagerResources { #[cfg(test)] instances: instances.clone(), pool: pool.clone(), metrics: metrics.clone(), }; - ( + Ok(( Self { instances, backend_instances, operation_locks: Mutex::new(HashMap::new()), pool, - spawners: Arc::new(spawners), + runtime_pool, + spawners, active_backend, storage, state_dir, rootfs_size, mem_size, metrics, - cancellation: CancellationToken::new(), + cancellation, }, resources, - ) + )) } /// Return the async operation lock that serializes one sandbox mutation. @@ -158,6 +199,23 @@ impl SandboxManager { } } + #[cfg(test)] + pub(crate) fn runtime_pool_status(&self) -> RuntimePoolStatus { + self.runtime_pool.status() + } + + #[cfg(test)] + pub(crate) fn runtime_pool_has_tracked_worker(&self) -> bool { + self.runtime_pool.has_tracked_worker() + } + + #[cfg(test)] + pub(crate) fn configure_runtime_pool_for_test(&self, prototype: PoolPrototype) -> Result { + self.runtime_pool + .configure(prototype) + .map_err(BlazeDaemonError::from) + } + /// Hold the per-sandbox mutation lock after confirming that the persisted /// record is in `expected_state` and has no unfinished operation. /// @@ -288,6 +346,30 @@ impl SandboxManager { request.image_digest.clone(), ); if request.decision.pool_eligible { + let warm_ttl = request + .decision + .pool + .as_ref() + .and_then(|pool| pool.warm_ttl.as_deref()) + .and_then(parse_duration) + .unwrap_or_else(|| self.runtime_pool.default_warm_ttl()); + let configured = self.runtime_pool.configure(PoolPrototype { + image_digest: request.image_digest.clone(), + policy_name: request.decision.policy_name.clone(), + workload_class: request.decision.workload_class, + templates: request.decision.templates.clone(), + kernel_hooks: request.decision.kernel_hooks.clone(), + binary_path: request.binary_path.clone(), + runtime_backend: request.runtime_backend, + backend: request.decision.backend.clone(), + vm: request.decision.vm.clone(), + warm_ttl, + })?; + if configured && let Some(lease) = self.runtime_pool.acquire().await? { + let result = self.activate_runtime_slot(lease, request).await?; + self.metrics.inc(&self.metrics.pool_hits); + return Ok(result); + } if let Some(result) = self.activate_warm(&pool_key).await? { self.metrics.inc(&self.metrics.pool_hits); return Ok(result); @@ -333,7 +415,7 @@ impl SandboxManager { }; crate::failpoint::pause("create-after-storage-acquire").await; - let run_dir = self.state_dir.join(instance.id.to_string()); + let run_dir = self.runtime_dir_for(&instance); let spawner = match self.spawners.get(self.active_backend) { Some(spawner) => spawner, None => { @@ -482,6 +564,319 @@ impl SandboxManager { }) } + async fn activate_runtime_slot( + &self, + mut lease: RuntimePoolLease, + request: CreateSandbox, + ) -> Result { + { + let slot = lease.slot()?; + if slot.runtime_backend != request.runtime_backend { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} records backend {} instead of requested {}", + slot.instance_id, slot.runtime_backend, request.runtime_backend + ))); + } + let expected_ownership = if slot.backend.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::NotStarted + }; + if slot.backend_ownership != expected_ownership { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} has backend ownership {:?} with backend handle present={}", + slot.instance_id, + slot.backend_ownership, + slot.backend.is_some() + ))); + } + } + + let owner_token = Uuid::new_v4(); + let instance_id = lease.slot()?.instance_id; + let operation_lock = self.operation_lock(instance_id); + let _operation = operation_lock.lock().await; + lease.begin_handoff(owner_token).await?; + let slot = lease.slot()?; + let reconstructed = self + .storage + .reconstruct(&slot.instance_id.to_string()) + .await?; + if reconstructed != slot.storage { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} storage paths changed before lifecycle handoff", + slot.instance_id + ))); + } + + let mut instance = SandboxInstance::new_warm_claim( + slot.instance_id, + slot.runtime_backend, + request.decision.workload_class, + request.image_digest, + request.decision.policy_name, + slot.backend_ownership, + owner_token, + )?; + let publication = instance + .persist(&self.state_dir) + .map_err(BlazeDaemonError::from) + .and_then(|()| crate::failpoint::state("warm-runtime-lifecycle-publish-result")); + let publication_failure = match publication { + Ok(()) => None, + Err(error) => match inspect_warm_claim_publication(&self.state_dir, &instance) { + WarmClaimPublication::Absent => return Err(error), + WarmClaimPublication::Published(published) => { + instance = published; + Some(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} lifecycle publication reported an error after state \ + became visible: {error}", + instance.id + ))) + } + WarmClaimPublication::Ambiguous(reason) => { + let unresolved = + format!("lifecycle publication was ambiguous after {error}: {reason}"); + lease.retain_unresolved(owner_token, unresolved.clone())?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} {unresolved}; ownership remains counted in the runtime \ + pool and requires startup reconciliation", + instance.id + ))); + } + }, + }; + let retained_state = self.retain_instance(instance.clone()); + let retained_backend = slot + .backend + .as_ref() + .and_then(|backend| self.retain_backend(instance.id, backend.clone())); + let mut registered = slot.backend.is_some(); + lease.transfer_to_lifecycle(); + + let handoff = lease.finish_handoff(owner_token).await; + let mut slot = lease.into_slot()?; + let mut transfer_errors = [retained_state, retained_backend] + .into_iter() + .flatten() + .collect::>(); + if let Err(error) = handoff { + transfer_errors.push(format!("finish durable ownership handoff: {error}")); + } + if let Some(error) = publication_failure { + transfer_errors.push(error.to_string()); + } + if !transfer_errors.is_empty() { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} lifecycle handoff failed: {}", + instance.id, + transfer_errors.join("; ") + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + + let actual_backend = if let Some(backend) = slot.backend.as_ref() { + backend.backend() + } else { + let spawner = match self.spawners.get(instance.backend) { + Some(spawner) => spawner, + None => { + let failure = BlazeDaemonError::Internal(format!( + "runtime backend {} has no registered spawner", + instance.backend + )); + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, failure) + .await); + } + }; + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance.persist(&self.state_dir) { + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, error.into()) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + None, + false, + BlazeDaemonError::Internal(error), + ) + .await); + } + if let Err(error) = spawner.prepare_spawn(&slot.run_dir).await { + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, None, false, error.into()) + .await); + } + let spawn = spawner + .spawn(SpawnRequest { + instance_id: instance.id, + run_dir: slot.run_dir.clone(), + binary_path: request.binary_path, + storage: slot.storage.clone(), + backend: request.decision.backend, + vm: request.decision.vm, + }) + .await; + let backend = match spawn { + Ok(backend) => backend, + Err(error) => { + let (source, owner) = error.into_parts(); + instance.backend_ownership = if owner.is_some() { + BackendOwnership::Running + } else { + BackendOwnership::Stopped + }; + let persist = instance.persist(&self.state_dir).err(); + let retained = self.retain_instance(instance.clone()); + let mut details = Vec::new(); + if let Some(error) = persist { + details.push(format!("backend failure state persistence failed: {error}")); + } + if let Some(error) = retained { + details.push(error); + } + let original = if details.is_empty() { + source.into() + } else { + BlazeDaemonError::RecoveryRequired(format!( + "{source}; {}", + details.join("; ") + )) + }; + return Err(self + .cleanup_failed_create(&mut instance, slot.storage, owner, false, original) + .await); + } + }; + let actual_backend = backend.backend(); + instance.backend_ownership = BackendOwnership::Running; + let persist = instance.persist(&self.state_dir); + let retained_state = self.retain_instance(instance.clone()); + let retained_backend = self.retain_backend(instance.id, backend.clone()); + registered = true; + slot.backend = Some(backend.clone()); + if actual_backend != instance.backend { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} expected backend {} but spawner returned {}", + instance.id, instance.backend, actual_backend + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + if let Err(error) = persist { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + if let Some(error) = retained_state.or(retained_backend) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + BlazeDaemonError::Internal(error), + ) + .await); + } + if let Err(error) = self + .wait_for_guest_ready(&backend, "warm-runtime-guest-ready") + .await + { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + actual_backend + }; + + if actual_backend != instance.backend { + let failure = BlazeDaemonError::RecoveryRequired(format!( + "runtime slot {} expected backend {} but owner is {}", + instance.id, instance.backend, actual_backend + )); + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + failure, + ) + .await); + } + if let Err(error) = instance.transition(SandboxState::Running) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + instance.finish_operation(); + if let Err(error) = instance.persist(&self.state_dir) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + error.into(), + ) + .await); + } + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(self + .cleanup_failed_create( + &mut instance, + slot.storage, + slot.backend, + registered, + BlazeDaemonError::Internal(error), + ) + .await); + } + Ok(CreateSandboxResult { + instance, + selected_backend: actual_backend, + }) + } + async fn activate_warm(&self, key: &PoolKey) -> Result> { let candidate = self.pool.lock().map_err(|_| poisoned("pool"))?.lookup(key); let Some(id) = candidate else { @@ -675,7 +1070,7 @@ impl SandboxManager { } None => match self.spawners.get(metadata.backend) { Some(spawner) => match spawner - .cleanup_orphan(id, &self.state_dir.join(id.to_string())) + .cleanup_orphan(id, &self.runtime_dir_for(metadata)) .await { Ok(()) => true, @@ -759,8 +1154,23 @@ impl SandboxManager { .map_err(|_| poisoned("backend_instances"))? .contains_key(&id); if is_clean_terminal(&original) && !has_retained_backend { + if original.runtime_location == RuntimeLocation::WarmPool { + let owner_token = required_runtime_owner_token(&original)?; + remove_lifecycle_tombstone(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: finish runtime tombstone cleanup: {error}" + )) + })?; + } return Ok(false); } + if original.runtime_location == RuntimeLocation::WarmPool { + self.cleanup_warm_runtime_locked(&mut original).await?; + self.metrics.inc(&self.metrics.instances_destroyed); + return Ok(true); + } if original.operation.as_ref().map(|operation| operation.kind) != Some(OperationKind::Destroy) @@ -803,7 +1213,7 @@ impl SandboxManager { match self.spawners.get(original.backend) { Some(spawner) => { spawner - .cleanup_orphan(id, &self.state_dir.join(id.to_string())) + .cleanup_orphan(id, &self.runtime_dir_for(&original)) .await } None => Err(BlazeError::BackendError { @@ -849,6 +1259,7 @@ impl SandboxManager { storage retained" ))); } + self.forget_backend(id); if let Err(error) = self.storage.release_by_id(&id.to_string()).await { let recovery = self.mark_recovery(id).err(); @@ -884,29 +1295,151 @@ impl SandboxManager { "destroy {id}: resources released but {error}" ))); } - match self.backend_instances.lock() { - Ok(mut instances) => { - instances.remove(&id); - } - Err(poisoned) => { - poisoned.into_inner().remove(&id); - } - } self.metrics.inc(&self.metrics.instances_destroyed); Ok(true) } + async fn cleanup_warm_runtime_locked(&self, instance: &mut SandboxInstance) -> Result<()> { + let id = instance.id; + instance.begin_destroy_recovery(); + crate::failpoint::state("destroy-intent-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: {error}; resources retained" + ))); + } + + let owner_token = required_runtime_owner_token(instance)?; + begin_lifecycle_cleanup( + &self.state_dir.join("runtime-pool"), + id, + instance.backend, + owner_token, + ) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: persist cleanup ownership: {error}" + )) + })?; + + let backend = self + .backend_instances + .lock() + .map_err(|_| poisoned("backend_instances"))? + .get(&id) + .cloned(); + crate::failpoint::backend("destroy-kill")?; + if let Some(backend) = backend.as_ref() { + backend.kill().await?; + } else if !matches!( + instance.backend_ownership, + BackendOwnership::NotStarted | BackendOwnership::Stopped + ) { + let spawner = self.spawners.get(instance.backend).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: no recovery spawner registered for {}", + instance.backend + )) + })?; + spawner + .cleanup_orphan(id, &self.runtime_dir_for(instance)) + .await?; + } + + instance.backend_ownership = BackendOwnership::Stopped; + crate::failpoint::state("destroy-stop-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: backend stopped but {error}; storage retained" + ))); + } + self.forget_backend(id); + + self.storage.release_by_id(&id.to_string()).await?; + if instance.state != SandboxState::Destroyed { + instance.transition(SandboxState::Destroyed)?; + } + crate::failpoint::state("destroy-released-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: resources released but {error}" + ))); + } + + tombstone_lifecycle_slot(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: tombstone runtime directory: {error}" + )) + })?; + instance.finish_operation(); + crate::failpoint::state("destroy-final-state-commit")?; + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance.clone()) { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: lifecycle is terminal but {error}" + ))); + } + remove_lifecycle_tombstone(&self.state_dir.join("runtime-pool"), id, owner_token) + .await + .map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cleanup warm runtime {id}: remove runtime tombstone: {error}" + )) + }) + } + /// Reconcile every record that is not cleanly terminal without aborting /// on one failure. pub async fn reconcile_startup(&self) -> ReconcileReport { self.cleanup_owned_instances().await } + /// Release unclaimed runtime slots before the daemon starts listening. + /// + /// Any ambiguous ownership record stops startup so the daemon never + /// serves an incomplete resource inventory. + pub async fn reconcile_runtime_pool_startup(&self) -> Result { + let durable_owners = match self.instances.lock() { + Ok(instances) => instances + .values() + .map(|instance| (instance.id, DurableRuntimeOwner::from(instance))) + .collect(), + Err(poisoned) => poisoned + .into_inner() + .values() + .map(|instance| (instance.id, DurableRuntimeOwner::from(instance))) + .collect(), + }; + reconcile_runtime_slots( + &self.state_dir.join("runtime-pool"), + &durable_owners, + self.storage.as_ref(), + self.spawners.as_ref(), + ) + .await + } + /// Cancel readiness polling before the daemon drains active requests. pub fn begin_shutdown(&self) { + self.runtime_pool.begin_shutdown(); self.cancellation.cancel(); } + /// Stop background capacity work and release pool-owned slots within the + /// daemon's shared runtime cleanup deadline. + pub async fn shutdown_runtime_pool_until(&self, deadline: tokio::time::Instant) -> Result<()> { + self.runtime_pool + .shutdown_until(deadline) + .await + .map_err(BlazeDaemonError::from) + } + /// Release every lifecycle record and retained backend owner, continuing /// after individual failures. pub async fn cleanup_owned_instances(&self) -> ReconcileReport { @@ -956,6 +1489,10 @@ impl SandboxManager { .await } + fn runtime_dir_for(&self, instance: &SandboxInstance) -> PathBuf { + derive_runtime_dir(&self.state_dir, instance.runtime_location, instance.id) + } + /// Release owned instances with a caller-supplied per-instance deadline. pub async fn cleanup_owned_instances_with_timeout( &self, @@ -1014,7 +1551,7 @@ impl SandboxManager { "instance cleanup exceeded {} ms per-record deadline", item_timeout.as_millis() ); - let recovery = self.mark_recovery(id).err(); + let recovery = self.mark_recovery_if_nonterminal(id).err(); report.failures.push(ReconcileFailure { instance_id: id, error: match recovery { @@ -1026,7 +1563,7 @@ impl SandboxManager { }); } Ok(Err(error)) => { - let recovery = self.mark_recovery(id).err(); + let recovery = self.mark_recovery_if_nonterminal(id).err(); report.failures.push(ReconcileFailure { instance_id: id, error: match recovery { @@ -1050,6 +1587,11 @@ impl SandboxManager { registered: bool, original: BlazeDaemonError, ) -> BlazeDaemonError { + if instance.runtime_location == RuntimeLocation::WarmPool { + return self + .cleanup_failed_warm_create(instance, storage, backend, registered, original) + .await; + } if instance.operation.is_none() && let Err(error) = instance.begin_operation(OperationKind::Create) { @@ -1143,6 +1685,54 @@ impl SandboxManager { )) } + async fn cleanup_failed_warm_create( + &self, + instance: &mut SandboxInstance, + storage: StorageSlot, + backend: Option, + registered: bool, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + let id = instance.id; + let has_retained_backend = match self.backend_instances.lock() { + Ok(instances) => instances.contains_key(&id), + Err(poisoned) => poisoned.into_inner().contains_key(&id), + }; + if !has_retained_backend { + match backend { + Some(backend) => { + if let Some(error) = self.retain_backend(id, backend) { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: {error}" + )); + } + } + None if registered => { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: registered backend owner is missing" + )); + } + None => {} + } + } + if storage.id != id.to_string() { + return BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: warm storage owner {} does not match lifecycle ID", + storage.id + )); + } + + match self.cleanup_warm_runtime_locked(instance).await { + Ok(()) => { + self.metrics.inc(&self.metrics.instances_destroyed); + original + } + Err(error) => BlazeDaemonError::RecoveryRequired(format!( + "{original}; instance {id}: cleanup incomplete: {error}" + )), + } + } + fn retain_failed_acquire( &self, instance: &mut SandboxInstance, @@ -1203,6 +1793,15 @@ impl SandboxManager { self.mark_instance_recovery(self.get(id)?) } + fn mark_recovery_if_nonterminal(&self, id: Uuid) -> Result<()> { + let instance = self.get(id)?; + if instance.is_clean_terminal() { + Ok(()) + } else { + self.mark_instance_recovery(instance) + } + } + fn mark_instance_recovery(&self, mut instance: SandboxInstance) -> Result<()> { if instance.state == SandboxState::Destroyed { if instance.operation.as_ref().map(|operation| operation.kind) @@ -1238,6 +1837,17 @@ impl SandboxManager { } } + fn forget_backend(&self, id: Uuid) { + match self.backend_instances.lock() { + Ok(mut instances) => { + instances.remove(&id); + } + Err(poisoned) => { + poisoned.into_inner().remove(&id); + } + } + } + fn retain_instance(&self, instance: SandboxInstance) -> Option { match self.instances.lock() { Ok(mut instances) => { @@ -1257,12 +1867,89 @@ fn poisoned(name: &str) -> BlazeDaemonError { } fn is_clean_terminal(instance: &SandboxInstance) -> bool { - instance.state == SandboxState::Destroyed - && instance.operation.is_none() - && matches!( - instance.backend_ownership, - BackendOwnership::NotStarted | BackendOwnership::Stopped - ) + instance.is_clean_terminal() +} + +enum WarmClaimPublication { + Absent, + Published(SandboxInstance), + Ambiguous(String), +} + +fn inspect_warm_claim_publication( + state_dir: &Path, + expected: &SandboxInstance, +) -> WarmClaimPublication { + let owner_dir = state_dir.join(expected.id.to_string()); + match std::fs::symlink_metadata(&owner_dir) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return WarmClaimPublication::Absent; + } + Ok(metadata) if !metadata.file_type().is_dir() => { + return WarmClaimPublication::Ambiguous(format!( + "{} is not a real lifecycle owner directory", + owner_dir.display() + )); + } + Ok(_) => {} + Err(error) => { + return WarmClaimPublication::Ambiguous(format!( + "inspect {}: {error}", + owner_dir.display() + )); + } + } + + let path = owner_dir.join("state.json"); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return WarmClaimPublication::Ambiguous(format!( + "{} has no lifecycle state file", + owner_dir.display() + )); + } + Ok(metadata) if !metadata.file_type().is_file() => { + return WarmClaimPublication::Ambiguous(format!( + "{} is not a regular lifecycle file", + path.display() + )); + } + Ok(_) => {} + Err(error) => { + return WarmClaimPublication::Ambiguous(format!("inspect {}: {error}", path.display())); + } + } + + let published = match SandboxInstance::load(state_dir, expected.id) { + Ok(published) => published, + Err(error) => { + return WarmClaimPublication::Ambiguous(format!("load {}: {error}", path.display())); + } + }; + let expected_value = serde_json::to_value(expected); + let published_value = serde_json::to_value(&published); + match (expected_value, published_value) { + (Ok(expected_value), Ok(published_value)) if expected_value == published_value => { + WarmClaimPublication::Published(published) + } + (Err(error), _) | (_, Err(error)) => WarmClaimPublication::Ambiguous(format!( + "compare visible lifecycle state {}: {error}", + path.display() + )), + _ => WarmClaimPublication::Ambiguous(format!( + "{} does not match the claimed runtime owner", + path.display() + )), + } +} + +fn required_runtime_owner_token(instance: &SandboxInstance) -> Result { + instance.runtime_owner_token.ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "warm runtime {} has no durable ownership token", + instance.id + )) + }) } fn operation_lock( @@ -1301,4 +1988,37 @@ mod operation_lock_tests { assert_eq!(locks.len(), 1); } + + #[cfg(unix)] + #[test] + fn linked_owner_directory_is_an_ambiguous_publication() { + use std::os::unix::fs::symlink; + + let state = tempfile::tempdir().expect("state root"); + let external = tempfile::tempdir().expect("external owner"); + let expected = SandboxInstance::new_warm_claim( + Uuid::new_v4(), + BackendKind::Mock, + blaze_core::policy::WorkloadClass::AgentTool, + "sha256:test".to_string(), + "test-policy".to_string(), + BackendOwnership::NotStarted, + Uuid::new_v4(), + ) + .expect("warm claim"); + expected + .persist(external.path()) + .expect("external lifecycle state"); + symlink( + external.path().join(expected.id.to_string()), + state.path().join(expected.id.to_string()), + ) + .expect("linked owner directory"); + + assert!(matches!( + inspect_warm_claim_publication(state.path(), &expected), + WarmClaimPublication::Ambiguous(reason) + if reason.contains("not a real lifecycle owner directory") + )); + } } diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index e41ed62617..2808449467 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -160,6 +160,10 @@ pub trait BackendSpawner: Send + Sync { /// Clean up a backend process and resources whose in-memory handle was /// lost across daemon restart. + /// + /// Implementations must be safe to retry after a prior successful call. + /// They may update backend-specific files, but must not remove `run_dir` + /// because the caller retains its durable ownership journal there. async fn cleanup_orphan(&self, instance_id: Uuid, run_dir: &Path) -> Result<()>; } diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index be8b5cf5b1..44f2a759a9 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -72,7 +72,11 @@ impl ServerState { state_dir: state_dir.clone(), rootfs_size: config.storage.rootfs_size, mem_size: config.storage.mem_size, - }); + pool_size: config.storage.pool_size, + prefork: config.storage.prefork, + default_warm_ttl: config.pool.default_warm_ttl.clone(), + gc_interval: config.pool.gc_interval.clone(), + })?; Ok(Self { config: Mutex::new(config), @@ -140,6 +144,22 @@ fn scan_state_dir(state_dir: &Path) -> Result> { mod tests { use super::*; + #[test] + fn scan_ignores_runtime_pool_root() { + let temp = tempfile::tempdir().expect("tempdir"); + let runtime_slot = temp + .path() + .join("runtime-pool") + .join(Uuid::new_v4().to_string()); + std::fs::create_dir_all(&runtime_slot).expect("runtime slot"); + std::fs::write(runtime_slot.join("ownership.json"), b"{not-lifecycle-state") + .expect("ownership marker"); + + let instances = scan_state_dir(temp.path()).expect("scan state"); + + assert!(instances.is_empty()); + } + #[test] fn scan_rejects_corrupt_owned_state() { let temp = tempfile::tempdir().expect("tempdir"); diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 1b1bc127d7..aa26000dfd 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -96,25 +96,27 @@ provider = "file" rootfs_size = 8589934592 mem_size = 4294967296 -# Number of pre-warmed storage slots to keep ready in the pool. -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) +# Number of recoverable runtime slots to keep ready for compatible requests. +# Zero disables background construction. Default: 0. +pool_size = 0 -# Whether to prefork (pre-allocate) storage slots at daemon startup. -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) +# Start each slot's backend before it becomes ready. When false, the slot owns +# storage only and starts its backend after a request claims it. Default: false. +prefork = false # Interval between flush-dirty sweeps. # flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) # --------------------------------------------------------------------------- -# Instance pool settings (global defaults for warm-pool recycling). -# Per-workload pool settings are configured in each policy file. +# Runtime slot maintenance defaults. +# A policy-specific [pool].warm_ttl overrides the daemon-wide TTL. # --------------------------------------------------------------------------- [pool] -# Default time-to-live for warm (idle) instances before the GC reclaims them. +# Default time-to-live for ready runtime slots before the worker reclaims them. # Format: duration string (e.g. "30m", "1h", "90s"). Default: "30m". default_warm_ttl = "30m" -# How often the garbage collector runs to reap expired warm instances. +# How often the runtime worker checks for expired slots and missing capacity. # Format: duration string. Default: "5m". gc_interval = "5m" From c968a7f076c96452e2aca7dded34541e7c5ee293 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 06:53:03 +0800 Subject: [PATCH 24/40] docs(blaze): document runtime slot ownership Operators need to configure background runtime capacity without confusing it with the lifecycle pool contract or storage health status. Add paired user guides and README summaries for eligibility, asynchronous construction, prefork behavior, TTL, capacity accounting, restart, and shutdown. Correct example policy comments so reserved sizing and reset fields are not presented as active runtime-slot controls. Document the ownership handoff and failure decisions with request, state, and startup flows. This enables operators and reviewers to distinguish implemented runtime-slot behavior from unconnected lifecycle return and management paths. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- docs/user-guide/en/README.md | 3 +- .../user-guide/en/runtime/blaze/QUICKSTART.md | 215 +++++++++++++ docs/user-guide/zh/README.md | 3 +- .../user-guide/zh/runtime/blaze/QUICKSTART.md | 200 +++++++++++++ src/blaze/README.md | 94 ++++-- src/blaze/README_zh.md | 84 +++++- .../docs/design/runtime-slot-ownership.md | 283 ++++++++++++++++++ src/blaze/examples/policies/agent-rl.toml | 23 +- src/blaze/examples/policies/agent-tool.toml | 22 +- 9 files changed, 875 insertions(+), 52 deletions(-) create mode 100644 docs/user-guide/en/runtime/blaze/QUICKSTART.md create mode 100644 docs/user-guide/zh/runtime/blaze/QUICKSTART.md create mode 100644 src/blaze/docs/design/runtime-slot-ownership.md diff --git a/docs/user-guide/en/README.md b/docs/user-guide/en/README.md index 7b63a6c4d0..5833807a93 100644 --- a/docs/user-guide/en/README.md +++ b/docs/user-guide/en/README.md @@ -14,7 +14,7 @@ ANOLISA provides a complete server-side runtime for AI Agent workloads. Componen │ anolisa-cli · cosh · os-skills │ ├──────────────────────────────────┬─────────────────────────────────┤ │ Token Saving │ Runtime │ -│ tokenless · agent-memory │ skillfs · ws-ckpt │ +│ tokenless · agent-memory │ blaze · skillfs · ws-ckpt │ ├──────────────────────────────────┼─────────────────────────────────┤ │ Agent Observability │ Agent Security │ │ agentsight │ agent-sec-core │ @@ -68,6 +68,7 @@ ANOLISA provides a complete server-side runtime for AI Agent workloads. Componen | Document | Component | Description | |----------|-----------|-------------| +| [Blaze Runtime Slots](runtime/blaze/QUICKSTART.md) | blaze | Bounded background storage slots with optional backend prefork | | [Workspace Checkpoints](runtime/ws-ckpt.md) | ws-ckpt | Instant snapshot/rollback via btrfs COW | | [Skill Filesystem](runtime/skillfs.md) | skillfs | FUSE virtual views with progressive disclosure | diff --git a/docs/user-guide/en/runtime/blaze/QUICKSTART.md b/docs/user-guide/en/runtime/blaze/QUICKSTART.md new file mode 100644 index 0000000000..87b2708053 --- /dev/null +++ b/docs/user-guide/en/runtime/blaze/QUICKSTART.md @@ -0,0 +1,215 @@ +# Blaze Runtime Slots + +[中文版](../../../zh/runtime/blaze/QUICKSTART.md) + +Blaze can prepare independent runtime slots in the background so a compatible +sandbox create can reuse prepared storage and, optionally, an already started +backend. The feature is bounded, disabled by default, and continues through +the existing create flow whenever no compatible slot is ready. + +## Requirements + +- Linux with root privileges for the selected sandbox backend +- Rust 1.88 or newer for source builds +- a Blaze policy whose `[pool]` section sets `enabled = true` +- stable daemon state, storage-instance, and runtime directories across restart + +## Installation + +### ANOLISA CLI + +Blaze is a Labs component. The source tree contains an ANOLISA component +manifest, but a configured component repository may not publish a `blaze` +candidate. Preview resolution before applying the system installation: + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +### RPM + +On an RPM repository that publishes Blaze: + +```bash +sudo yum install blaze +``` + +### Source build + +```bash +cd src/blaze +cargo build --release --locked +``` + +## Enable Background Capacity + +Set a non-zero target in the daemon configuration: + +```toml +[storage] +provider = "file" +images_dir = "/var/lib/blaze/images" +instances_dir = "/var/lib/blaze/instances" +pool_size = 2 +prefork = false + +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +Enable eligibility in each policy that may use a prepared slot: + +```toml +[pool] +enabled = true +# Optional. When omitted, pool.default_warm_ttl from config.toml applies. +warm_ttl = "15m" +``` + +`storage.pool_size` is the target for background runtime slots. Policy `min`, +`target`, `max`, and `reset_mode` are reserved policy schema metadata. The +runtime-slot worker does not consume them, and `/v1/pools` does not apply them +from policy. None of them resizes background runtime capacity. The public reset +operation currently returns `501`, so a complete lifecycle return-to-pool +workflow is not connected. + +`pool_size = 0` disables construction. Duration values must include a positive +unit: `s`, `m`, `h`, or `d`. + +## Start Blaze + +Use the packaged service: + +```bash +sudo systemctl enable --now blazed +``` + +For a source checkout, run the built daemon with a configuration whose +`policy.dir` points to readable policies: + +```bash +sudo ./target/release/blazed daemon start --config examples/config.toml +``` + +The example configuration points `policy.dir` to +`/etc/anolisa/blaze/policies`. A package installs policies there; for a source +checkout, copy the example policies to that directory or edit the configuration +to use the checkout path. + +## Create and Claim a Slot + +The first eligible create fixes one build shape for this daemon run and wakes +the background worker. It does not wait for the worker to fill the target, so +that request normally continues through the existing create flow. Later +compatible requests claim a ready slot when one is available: + +```bash +curl -X POST --unix-socket /run/blaze/api.sock \ + http://localhost/v1/sandboxes \ + -H 'Content-Type: application/json' \ + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' +``` + +The existing `start_path` field is a generic warm-start classification. A +background runtime-slot claim reports this shape: + +```json +{ + "start_path": "warm", + "instance": { + "start_path": "warm", + "runtime_location": "warm-pool" + } +} +``` + +A `"cold"` result is not an error: this request did not use an applicable warm +source. + +```mermaid +flowchart LR + A["POST /v1/sandboxes"] --> B{"Policy eligible?"} + B -- "No" --> C["Existing create flow"] + B -- "Yes" --> D["Configure or match build shape"] + D --> K{"Prototype accepted?"} + K -- "No" --> C + K -- "Yes" --> E{"Compatible slot ready?"} + E -- "No" --> C + E -- "Yes" --> F["Record ownership handoff"] + F --> G["Publish lifecycle owner"] + G --> H["Return Running sandbox"] + K -- "Yes" --> I["Wake background worker"] + I --> J["Build toward pool_size"] +``` + +## Prefork Modes + +| `storage.prefork` | Prepared slot | Work performed after claim | +| --- | --- | --- | +| `false` | Independent storage | Start the backend and wait for guest readiness when the backend exposes a guest endpoint | +| `true` | Independent storage plus a running backend | Check backend liveness at claim; when a guest endpoint exists, readiness was already checked before the slot became ready | + +Every slot owns its own storage snapshot. A slot is never made ready by sharing +another sandbox's mutable storage. + +## Capacity and Expiry + +The target counts ready slots, active builds, leases being handed off, and +pool-owned slots awaiting cleanup. An ambiguous handoff has no selected +cleanup owner, but remains accounted for until reconciliation. A sandbox stops +consuming this target after lifecycle ownership is durably established. + +The worker removes ready slots after their effective `warm_ttl`. Claim checks +expiry for every slot and backend liveness when the slot holds a prefork +backend. Pool cleanup failures remain owned and are retried; they continue to +consume target capacity until cleanup succeeds. + +## Restart and Shutdown + +Before opening its API listeners, Blaze reconciles every runtime-slot ownership +record with durable sandbox lifecycle state. It cleans unclaimed slots instead +of rebuilding the old ready queue. Any ambiguous or inconsistent record, or +runtime reconciliation step that cannot complete, stops startup. After runtime +reconciliation succeeds, ordinary sandbox lifecycle reconciliation runs; one +sandbox cleanup failure is retained and reported but does not prevent listener +startup. + +Keep `daemon.state_dir`, `storage.instances_dir`, the selected storage provider, +and backend availability consistent across restart. Changing those values can +prevent the daemon from identifying and cleaning resources created by the +previous run. + +During graceful shutdown, Blaze stops new slot construction, joins the worker, +and attempts bounded cleanup for every pool-owned slot. Lifecycle-owned +sandboxes follow the normal sandbox cleanup path. + +## Current Boundaries + +- One compatible build shape is accepted per daemon run. Requests with another + image, backend, policy shape, or runtime configuration continue through the + existing create flow. +- Capacity starts on the first eligible create; daemon startup does not prefill + slots. +- Restart cleans unclaimed slots and does not restore them as ready. +- There is no public status, drain, or refill endpoint for background runtime + slots. `/v1/pools` and the health response's `storage_pool` object refer to + other pool contracts. +- A configured target improves the chance of a warm claim but does not + guarantee one for every request. +- A claimed background slot is single-use: destroy releases its resources and + the worker builds replacement capacity instead of returning that sandbox to + the ready queue. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Every response has `start_path: "cold"` | Confirm `pool_size > 0`, policy `enabled = true`, and identical request/build inputs; then allow time for background construction | +| Slot construction keeps retrying | Inspect daemon logs for storage, backend start, or guest-readiness errors | +| Capacity appears below target | Cleanup or an unresolved handoff may still count toward the target; inspect daemon logs | +| Daemon stops during startup reconciliation | Restore the provider and directory configuration used by the previous run, then inspect the reported ownership record | + +For the ownership and recovery rationale, see +[Runtime Slot Ownership](../../../../../src/blaze/docs/design/runtime-slot-ownership.md). diff --git a/docs/user-guide/zh/README.md b/docs/user-guide/zh/README.md index 371c72b26f..da1d0ce502 100644 --- a/docs/user-guide/zh/README.md +++ b/docs/user-guide/zh/README.md @@ -14,7 +14,7 @@ ANOLISA 为 AI Agent 提供完整的服务端运行时能力。通过 `anolisa` │ anolisa-cli · cosh · os-skills │ ├──────────────────────────────────┬─────────────────────────────────┤ │ Token 节省 │ 运行时 │ -│ tokenless · agent-memory │ skillfs · ws-ckpt │ +│ tokenless · agent-memory │ blaze · skillfs · ws-ckpt │ ├──────────────────────────────────┼─────────────────────────────────┤ │ Agent 可观测 │ Agent 安全 │ │ agentsight │ agent-sec-core │ @@ -68,6 +68,7 @@ ANOLISA 为 AI Agent 提供完整的服务端运行时能力。通过 `anolisa` | 文档 | 组件 | 说明 | |------|------|------| +| [Blaze Runtime 槽位](runtime/blaze/QUICKSTART.md) | blaze | 有界后台存储槽位，可选后端 prefork | | [工作区快照](runtime/ws-ckpt.md) | ws-ckpt | 秒级快照创建/回滚，基于 btrfs COW | | [技能文件系统](runtime/skillfs.md) | skillfs | FUSE 虚拟视图、渐进披露 | diff --git a/docs/user-guide/zh/runtime/blaze/QUICKSTART.md b/docs/user-guide/zh/runtime/blaze/QUICKSTART.md new file mode 100644 index 0000000000..083f80ab8d --- /dev/null +++ b/docs/user-guide/zh/runtime/blaze/QUICKSTART.md @@ -0,0 +1,200 @@ +# Blaze Runtime 槽位 + +[English](../../../en/runtime/blaze/QUICKSTART.md) + +Blaze 可以在后台准备彼此独立的 runtime 槽位，让兼容的 sandbox 创建请求 +复用已准备的存储，并可选择复用已经启动的后端。该功能有容量上限、默认 +关闭；没有兼容槽位就绪时，请求会继续走已有创建流程。 + +## 环境要求 + +- Linux；所选 sandbox 后端需要 root 权限 +- 从源码构建时需要 Rust 1.88 或更高版本 +- Blaze 策略的 `[pool]` 设置 `enabled = true` +- daemon 状态目录、存储 instance 目录和 runtime 目录在重启前后保持稳定 + +## 安装 + +### ANOLISA CLI + +Blaze 当前是 Labs 组件。源码树中包含 ANOLISA 组件清单，但配置的组件仓库 +不一定发布 `blaze` 候选包。执行系统级安装前，先预览解析结果： + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +### RPM + +如果 RPM 仓库发布了 Blaze： + +```bash +sudo yum install blaze +``` + +### 从源码构建 + +```bash +cd src/blaze +cargo build --release --locked +``` + +## 启用后台容量 + +在 daemon 配置中设置非零目标： + +```toml +[storage] +provider = "file" +images_dir = "/var/lib/blaze/images" +instances_dir = "/var/lib/blaze/instances" +pool_size = 2 +prefork = false + +[pool] +default_warm_ttl = "30m" +gc_interval = "5m" +``` + +在允许使用准备槽位的策略中启用该能力： + +```toml +[pool] +enabled = true +# 可选。省略时使用 config.toml 中的 pool.default_warm_ttl。 +warm_ttl = "15m" +``` + +`storage.pool_size` 是后台 runtime 槽位的目标。策略中的 `min`、`target`、 +`max`、`reset_mode` 是预留的 policy schema metadata。runtime-slot worker +不会读取它们，`/v1/pools` 也不会从 policy 自动应用这些值；它们都不会调整 +后台 runtime 容量。公开 reset 操作目前返回 `501`，因此完整的 lifecycle +return-to-pool 流程尚未接通。 + +`pool_size = 0` 会关闭构建。duration 必须是带正数单位的值：`s`、`m`、 +`h` 或 `d`。 + +## 启动 Blaze + +使用软件包提供的服务： + +```bash +sudo systemctl enable --now blazed +``` + +使用源码 checkout 时，确保配置中的 `policy.dir` 指向可读的策略目录，然后 +运行构建出的 daemon： + +```bash +sudo ./target/release/blazed daemon start --config examples/config.toml +``` + +示例配置中的 `policy.dir` 指向 `/etc/anolisa/blaze/policies`。软件包会在该 +目录安装策略；使用源码 checkout 时，应把示例策略复制到该目录，或者修改 +配置以使用 checkout 中的路径。 + +## 创建并取用槽位 + +首个符合条件的创建请求会固定本次 daemon 运行使用的一组构建参数，并唤醒 +后台 worker。该请求不会等待 worker 填满目标，因此通常会继续走已有创建 +流程。之后的兼容请求会在有槽位就绪时取用它： + +```bash +curl -X POST --unix-socket /run/blaze/api.sock \ + http://localhost/v1/sandboxes \ + -H 'Content-Type: application/json' \ + -d '{"workload_class":"agent-tool","image_digest":"sha256:..."}' +``` + +现有 `start_path` 字段是通用的 warm-start 分类。后台 runtime 槽位会返回 +以下结构： + +```json +{ + "start_path": "warm", + "instance": { + "start_path": "warm", + "runtime_location": "warm-pool" + } +} +``` + +`"cold"` 结果不是错误，只表示本次请求没有使用适用的 warm 来源。 + +```mermaid +flowchart LR + A["POST /v1/sandboxes"] --> B{"策略符合条件？"} + B -- "否" --> C["已有创建流程"] + B -- "是" --> D["配置或匹配构建参数"] + D --> K{"构建参数已接受？"} + K -- "否" --> C + K -- "是" --> E{"兼容槽位已就绪？"} + E -- "否" --> C + E -- "是" --> F["记录 ownership 交接"] + F --> G["发布 lifecycle owner"] + G --> H["返回 Running sandbox"] + K -- "是" --> I["唤醒后台 worker"] + I --> J["向 pool_size 补充"] +``` + +## Prefork 模式 + +| `storage.prefork` | 已准备的槽位 | 取用后仍需完成的工作 | +| --- | --- | --- | +| `false` | 独立存储 | 启动后端；后端提供 guest endpoint 时等待 guest readiness | +| `true` | 独立存储和运行中的后端 | 取用时检查后端存活；存在 guest endpoint 时，槽位进入 ready 前已经检查 readiness | + +每个槽位都持有自己的存储快照，不会通过共享另一个 sandbox 的可变存储进入 +ready。 + +## 容量与过期 + +目标会统计 ready 槽位、正在构建的槽位、正在交接的 lease，以及等待清理的 +pool-owned 槽位。未决交接没有选定 cleanup owner，但在核对完成前仍计入 +目标容量。生命周期 ownership 持久化成功后，该 sandbox 不再占用此目标。 + +ready 槽位超过有效 `warm_ttl` 后，worker 会将其清理。请求取用槽位时也会 +检查过期时间；只有槽位持有 prefork 后端时才检查后端存活。pool cleanup +失败的资源会继续被持有并重试；清理成功前，它们仍占用目标容量。 + +## 重启与关闭 + +Blaze 打开 API listener 前，会把每条 runtime 槽位 ownership 记录与 +sandbox 的持久化生命周期状态进行核对。它会清理未交接槽位，而不是重建 +旧的 ready 队列。存在无法明确归属或互相矛盾的记录，或者 runtime 核对 +步骤无法完成时，daemon 会停止启动。runtime 核对成功后才执行普通 sandbox +生命周期核对；其中单个 sandbox 清理失败会被保留和报告，但不会阻止 +listener 启动。 + +重启前后应保持 `daemon.state_dir`、`storage.instances_dir`、所选存储 +provider 和后端可用性一致。修改这些值可能导致 daemon 无法识别和清理 +上一次运行创建的资源。 + +正常关闭时，Blaze 会停止创建新槽位、等待 worker 结束，并在有界时间内 +尝试清理所有 pool-owned 槽位。lifecycle-owned sandbox 仍走普通 sandbox +清理路径。 + +## 当前边界 + +- 每次 daemon 运行只接受一组兼容构建参数。使用其他 image、backend、 + policy 参数或 runtime 配置的请求会继续走已有创建流程。 +- 容量由首个符合条件的创建请求启动；daemon 启动时不会预填充槽位。 +- 重启会清理未交接槽位，不会把它们恢复为 ready。 +- 后台 runtime 槽位目前没有公开的 status、drain 或 refill 接口。 + `/v1/pools` 和健康响应中的 `storage_pool` 对象属于其他 pool contract。 +- 配置目标只能提高 warm claim 的概率，不能保证每个请求都命中。 +- 取用的后台槽位只使用一次：destroy 会释放其资源，worker 重新构建补充 + 容量，不会把该 sandbox 放回 ready 队列。 + +## 故障排查 + +| 现象 | 检查项 | +| --- | --- | +| 所有响应都是 `start_path: "cold"` | 确认 `pool_size > 0`、策略 `enabled = true` 且请求和构建参数一致；然后为后台构建预留时间 | +| 槽位构建持续重试 | 查看 daemon 日志中的存储、后端启动或 guest readiness 错误 | +| 容量看起来低于目标 | 清理或未决交接可能仍占用目标；查看 daemon 日志 | +| daemon 在启动核对时停止 | 恢复上一次运行使用的 provider 和目录配置，再检查日志指出的 ownership 记录 | + +ownership 与恢复设计参见 +[Runtime 槽位 Ownership](../../../../../src/blaze/docs/design/runtime-slot-ownership.md)。 diff --git a/src/blaze/README.md b/src/blaze/README.md index 69b4a13035..ba1b43ef10 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -17,20 +17,46 @@ Designed as the per-host agent for E2B-style orchestrator platforms. Checkpointed, RecoveryRequired, Reset, Warm, and Destroyed - **Guest operations** — bounded command execution and file transfer for running backends that expose a guest endpoint -- **Warm pool management** — pre-warmed instances with TTL-based GC +- **Runtime slot capacity** — independent storage slots with optional backend + prefork and TTL-based cleanup - **Template registry** — in-memory template tracking with idle eviction - **Kernel hook registry** — state tracking for pre/post hooks - **Prometheus metrics** — request counts, instance gauges, pool sizes - **Spawners** — FirecrackerSpawner, BubblewrapSpawner, MockSpawner -## Quick Start +## Installation + +Blaze is a Labs component. This source tree contains its ANOLISA component +manifest and RPM packaging, but not every configured component repository +publishes a `blaze` candidate. Preview repository resolution before applying +the system installation: + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +On an RPM repository that publishes Blaze: + +```bash +sudo yum install blaze +``` + +For a developer source build: ```bash -# Build cd src/blaze -cargo build --release +cargo build --release --locked +``` + +## Quick Start + +```bash +# Choose one startup method; do not run both at the same time. +# Packaged installation +sudo systemctl enable --now blazed -# Run daemon (dev: override policy.dir to use local examples) +# Source build alternative (override policy.dir to use local examples) sudo ./target/release/blazed daemon start --config examples/config.toml # Note: the default config sets policy.dir = /etc/anolisa/blaze/policies. # For source-checkout testing, create a symlink or override: @@ -119,13 +145,34 @@ provider = "file" # Storage provider selection. Currently supported: "file # "auto" probes available providers in priority order (currently equivalent to "file"). # Other values will log a warning and fall back to file. images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] Warm pool slots (not yet active) -# prefork = false # [Reserved] Pre-start VMs in pool (not yet active) +pool_size = 0 # Background runtime slots; zero disables construction +prefork = false # Start the backend before a slot becomes ready # flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) + +[pool] +default_warm_ttl = "30m" # Used when an eligible policy omits warm_ttl +gc_interval = "5m" # Expiry and capacity maintenance interval ``` The `file` provider uses standard filesystem operations for sandbox storage. The `auto` provider probes available backends in priority order (currently equivalent to `file`). Unrecognized values will log a warning and fall back to `file`. +When `pool_size` is non-zero, the first eligible create request fixes one +compatible build shape and starts background construction. Every slot owns +storage; a slot owns a running backend only when `prefork` is enabled. +`pool_size` limits pool-owned and in-flight slots, not sandboxes that have +already completed lifecycle handoff. Incompatible requests continue through +the existing create flow. A policy is eligible only when its `[pool]` section +sets `enabled = true`; its optional `warm_ttl` overrides `default_warm_ttl`. + +On restart, the daemon cleans unclaimed slot journals before serving requests; +it does not restore old slots to the ready queue. That cleanup uses the +currently configured provider and storage/runtime roots, so those settings +must continue to identify the same owned directories across a restart. The +`/v1/pools` endpoints below expose a separate lifecycle recycling-pool +contract and do not expose this background runtime capacity. Public reset +currently returns `501`, so no production path returns a used sandbox to that +pool. + ## API Endpoints | Method | Path | Description | @@ -148,10 +195,10 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances/{id}/write` | Compatible guest file write action | | POST | `/v1/instances/{id}/checkpoint` | Reserved; returns `501` until backend and storage capture is implemented | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | -| GET | `/v1/pools` | List warm pools | -| GET | `/v1/pools/{backend}/{class}` | Get pool status | -| POST | `/v1/pools/{backend}/{class}/drain` | Drain a pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a pool | +| GET | `/v1/pools` | List lifecycle recycling pools | +| GET | `/v1/pools/{backend}/{class}` | Get lifecycle recycling-pool status | +| POST | `/v1/pools/{backend}/{class}/drain` | Drain a lifecycle recycling pool | +| PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a lifecycle recycling pool | | GET | `/v1/templates` | List templates | | GET | `/v1/templates/{id}` | Inspect a template | | POST | `/v1/templates/gc` | Trigger template GC | @@ -167,9 +214,11 @@ resources. A successful create finishes in `Running`; a successful destroy finishes in `Destroyed`. If compensation cannot release every owned resource, the sandbox remains visible as `RecoveryRequired` so destroy can be retried. -At startup, the daemon reconciles each non-terminal sandbox independently. -Failure to clean up one sandbox does not prevent the remaining records from -being processed or the API from starting. +Runtime-slot reconciliation completes before this lifecycle pass. An +inventory, journal, or runtime cleanup error stops startup. After it succeeds, +the daemon reconciles each non-terminal sandbox independently. Failure to +clean up one sandbox does not prevent the remaining records from being +processed or the API from starting. During graceful shutdown, the daemon first stops accepting work and drains accepted connections. It then attempts bounded cleanup for every persisted @@ -190,8 +239,10 @@ Guest operations are available only while a sandbox is `Running` and its backend reports a guest endpoint. A cold create that reports such an endpoint waits for the guest agent to answer before publishing `Running`. Backends with guest support disabled skip that wait, and later guest-operation requests -return HTTP 409. Warm-pool activation does not currently repeat the guest -readiness probe. +return HTTP 409. A prefork runtime waits for guest readiness before its slot +becomes ready when the backend exposes a guest endpoint. Claim checks backend +liveness without repeating guest readiness. A storage-only slot waits after it +starts its backend. Guest operations and lifecycle changes use the same per-sandbox operation lock. A request may wait for an earlier lifecycle action. After it obtains the @@ -229,7 +280,8 @@ Streaming files, interactive terminals, and session reuse are not supported. #### Health Check -`GET /v1/health` returns daemon status including storage pool readiness: +`GET /v1/health` returns daemon status including provider storage-pool +readiness: ```json { @@ -239,6 +291,13 @@ Streaming files, interactive terminals, and session reuse are not supported. } ``` +The `storage_pool` object does not report background runtime-slot capacity. + +## Documentation + +- [Runtime slot user guide](../../docs/user-guide/en/runtime/blaze/QUICKSTART.md) +- [Runtime slot ownership design](docs/design/runtime-slot-ownership.md) + ## Project Layout ``` @@ -246,6 +305,7 @@ src/blaze/ ├── crates/ │ ├── blaze-core/ # Library: policy, lifecycle, pool, template, kernel, config │ └── blazed/ # Binary: daemon, API server, spawners, metrics +├── docs/design/ # Component design documents ├── examples/ # config.toml, policies/ ├── dist/ # blazed.service, blaze.spec, tmpfiles └── manifests/ # Component metadata diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index df9d643e3e..085c629362 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -15,20 +15,44 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **生命周期状态机** — 9 种状态：Pending、Creating、Running、Paused、 Checkpointed、RecoveryRequired、Reset、Warm 和 Destroyed - **Guest 操作** — 对提供 guest endpoint 的运行中后端执行有界命令和文件传输 -- **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC +- **Runtime 槽位容量** — 独立存储槽位、可选后端 prefork 和基于 TTL 的清理 - **模板注册表** — 内存中模板追踪，支持空闲驱逐 - **内核 hook 注册** — 前/后置 hook 状态追踪 - **Prometheus 指标** — 请求计数、实例 gauge、池大小 - **Spawner 后端** — FirecrackerSpawner、BubblewrapSpawner、MockSpawner -## 快速开始 +## 安装 + +Blaze 当前是 Labs 组件。源码树中包含 ANOLISA 组件清单和 RPM 打包文件，但 +配置的组件仓库不一定发布 `blaze` 候选包。执行系统级安装前，先预览仓库 +解析结果： + +```bash +sudo anolisa --install-mode system --dry-run install blaze +sudo anolisa --install-mode system install blaze +``` + +如果 RPM 仓库发布了 Blaze： + +```bash +sudo yum install blaze +``` + +开发者从源码构建： ```bash -# 构建 cd src/blaze -cargo build --release +cargo build --release --locked +``` -# 运行 daemon（开发环境：覆盖 policy.dir 使用本地示例） +## 快速开始 + +```bash +# 选择一种启动方式，不要同时运行两种方式。 +# 软件包安装 +sudo systemctl enable --now blazed + +# 源码构建方式（覆盖 policy.dir 使用本地示例） sudo ./target/release/blazed daemon start --config examples/config.toml # 注意：默认配置设置 policy.dir = /etc/anolisa/blaze/policies。 # 源码开发测试时，创建符号链接或覆盖： @@ -115,13 +139,30 @@ provider = "file" # 存储 provider 选择。当前支持："file"、"auto # "auto" 按优先级探测可用 provider（当前等同于 "file"）。 # 其他值将记录告警并回退到 file。 images_dir = "/var/lib/blaze/images" -# pool_size = 0 # [Reserved] 预热存储槽位数（尚未启用） -# prefork = false # [Reserved] 是否在槽位中预启动 VM（尚未启用） +pool_size = 0 # 后台运行槽位数；0 表示不构建 +prefork = false # 槽位就绪前是否启动后端 # flush_interval = "30s" # [Reserved] 脏数据刷盘周期（尚未启用） + +[pool] +default_warm_ttl = "30m" # 符合条件的策略未设置 warm_ttl 时使用 +gc_interval = "5m" # 过期检查和容量维护间隔 ``` `file` provider 使用标准文件系统操作管理 sandbox 存储。`auto` 按优先级探测可用 provider（当前等同于 `file`）。无法识别的值将记录告警并回退到 `file`。 +`pool_size` 非零时，首个符合条件的创建请求会固定一组兼容构建参数，并启动 +后台补充。每个槽位都持有存储；只有启用 `prefork` 时，槽位才同时持有已经 +运行的后端。`pool_size` 限制池内及正在构建或交接的槽位，不限制已经完成 +生命周期交接的 sandbox。参数不兼容的请求会继续走已有创建流程。 +只有策略的 `[pool]` 设置 `enabled = true` 时，该策略才符合条件；策略中 +可选的 `warm_ttl` 会覆盖 `default_warm_ttl`。 + +daemon 重启时会在接收请求前清理尚未交接的槽位记录，不会把旧槽位重新放回 +ready 队列。该清理使用重启后当前配置的 provider 以及存储和运行目录，因此 +这些设置必须继续指向同一组已有目录。下文的 `/v1/pools` 描述的是另一套 +生命周期回收 pool 的管理接口，不展示这里的后台运行容量。公开 reset 当前 +返回 `501`，因此没有生产路径把已经使用的 sandbox 放回该 pool。 + ## API 端点 | 方法 | 路径 | 说明 | @@ -144,10 +185,10 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | | POST | `/v1/instances/{id}/checkpoint` | 预留接口；后端和存储快照实现前返回 `501` | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | -| GET | `/v1/pools` | 列出 warm pool | -| GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | -| POST | `/v1/pools/{backend}/{class}/drain` | 排空 pool | -| PUT | `/v1/pools/{backend}/{class}/sizing` | 调整 pool 大小 | +| GET | `/v1/pools` | 列出生命周期回收 pool | +| GET | `/v1/pools/{backend}/{class}` | 获取生命周期回收 pool 状态 | +| POST | `/v1/pools/{backend}/{class}/drain` | 排空生命周期回收 pool | +| PUT | `/v1/pools/{backend}/{class}/sizing` | 调整生命周期回收 pool 大小 | | GET | `/v1/templates` | 列出模板 | | GET | `/v1/templates/{id}` | 查看模板详情 | | POST | `/v1/templates/gc` | 触发模板 GC | @@ -162,8 +203,10 @@ images_dir = "/var/lib/blaze/images" `Running`，销毁成功后状态为 `Destroyed`。如果失败补偿不能释放全部已有 资源，sandbox 会保留为可查询的 `RecoveryRequired`，后续可以再次执行销毁。 -daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失败不会阻止 -其他记录继续处理，也不会阻止 API 启动。 +runtime 槽位核对会在该生命周期处理之前完成；inventory、journal 或 runtime +清理失败会停止启动。runtime 核对成功后，daemon 会逐个处理未结束的 +sandbox。单个 sandbox 清理失败不会阻止其他记录继续处理，也不会阻止 API +启动。 正常关闭时，daemon 会先停止接收新请求并等待已有连接结束，再为每条持久化 记录和仍持有的后端资源执行有界清理。单条清理失败不会跳过其余 sandbox， @@ -180,8 +223,9 @@ daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失 只有 sandbox 处于 `Running` 且后端报告了 guest endpoint 时，才能执行 guest 操作。冷启动后端如果报告了该 endpoint，创建流程会等待 guest agent 响应后才发布 `Running`。关闭 guest 支持的后端会跳过等待，后续 guest -操作返回 HTTP 409。当前从 warm pool 激活实例时不会再次执行 guest -readiness 探测。 +操作返回 HTTP 409。后端提供 guest endpoint 时，prefork 槽位会在进入 +ready 前等待 guest readiness；取用时再次检查后端存活，但不重复 guest +readiness。仅含存储的槽位会在启动后端后等待 guest readiness。 Guest 操作和生命周期变更使用同一个 sandbox 操作锁。请求可能等待先开始的 生命周期操作；取得锁后，manager 会再次检查 `Running`。如果 destroy 或 @@ -215,7 +259,7 @@ read 返回内容过大时，API 返回 HTTP 502 和 #### 健康检查 -`GET /v1/health` 返回 daemon 状态，包含存储池就绪信息： +`GET /v1/health` 返回 daemon 状态，包含 provider 存储池就绪信息： ```json { @@ -225,6 +269,13 @@ read 返回内容过大时，API 返回 HTTP 502 和 } ``` +`storage_pool` 对象不报告后台 runtime 槽位容量。 + +## 文档 + +- [Runtime 槽位用户指南](../../docs/user-guide/zh/runtime/blaze/QUICKSTART.md) +- [Runtime 槽位 ownership 设计](docs/design/runtime-slot-ownership.md) + ## 项目结构 ``` @@ -232,6 +283,7 @@ src/blaze/ ├── crates/ │ ├── blaze-core/ # 库：策略、生命周期、池、模板、内核、配置 │ └── blazed/ # 二进制：daemon、API server、spawner、指标 +├── docs/design/ # 组件设计文档 ├── examples/ # config.toml、policies/ ├── dist/ # blazed.service、blaze.spec、tmpfiles └── manifests/ # 组件元数据 diff --git a/src/blaze/docs/design/runtime-slot-ownership.md b/src/blaze/docs/design/runtime-slot-ownership.md new file mode 100644 index 0000000000..7cfeb48b25 --- /dev/null +++ b/src/blaze/docs/design/runtime-slot-ownership.md @@ -0,0 +1,283 @@ +# Runtime Slot Ownership + +## Purpose + +Background runtime capacity reduces the work on a compatible sandbox create +without allowing partially built storage or backend processes to lose an +owner. Each prepared slot has independent storage and may also own a running +backend. The daemon records who owns those resources before and during every +handoff, cleanup, restart, and shutdown transition. + +This design covers background construction, claim, ownership transfer, and +recovery. It does not add a public runtime-capacity management API, restore +ready slots after restart, or change the existing lifecycle recycling pool +served by `/v1/pools`. + +## Inputs and Activation + +The daemon reads these runtime-capacity settings: + +| Setting | Effect | +| --- | --- | +| `storage.pool_size` | Maximum pool-owned and in-flight slot count; zero disables construction | +| `storage.prefork` | Whether a slot starts its selected backend before becoming ready | +| `pool.default_warm_ttl` | Ready-slot lifetime when an eligible policy has no override | +| `pool.gc_interval` | Maintenance interval for expiry, cleanup, and refill work | +| policy `pool.enabled` | Whether creates evaluated by that policy may configure or claim a slot | +| policy `pool.warm_ttl` | Optional lifetime override for that policy's accepted build shape | + +The first eligible create fixes one prototype for the current daemon run. +Later creates must produce the same prototype fingerprint to claim its slots. +An incompatible request continues through the existing create flow and does +not replace the active prototype. + +## Components + +```mermaid +flowchart TB + API["Sandbox create API"] --> Manager["SandboxManager"] + Manager --> Pool["RuntimeWarmPool"] + Pool --> Storage["StorageProvider"] + Pool --> Spawners["SpawnerRegistry"] + Pool --> Journal["Runtime ownership journal"] + Manager --> State["Sandbox lifecycle state"] + Manager --> Spawners + Manager --> Storage + Journal <--> Recovery["Startup reconciler"] + State <--> Recovery + Recovery --> Storage + Recovery --> Spawners +``` + +- `RuntimeWarmPool` owns the bounded worker, ready queue, leases, and pending + cleanup. +- `StorageProvider` creates independent slots. Non-zero capacity requires a + provider that can enumerate every ID it owns and release an ID even when + creation stopped partway through. +- `SpawnerRegistry` starts the configured backend. During recovery, the + persisted backend kind selects its cleanup implementation; recovery does not + restart or adopt the old process. +- `SandboxManager` is the transfer boundary between pool ownership and normal + sandbox lifecycle ownership. +- The startup reconciler compares runtime journals, provider inventory, and + durable lifecycle state before the API starts listening. + +Provider- and backend-specific behavior remains behind the existing traits. +The pool records stable IDs and ownership facts; it does not persist a concrete +provider implementation or choose a replacement backend after failure. + +## Slot Construction + +```mermaid +sequenceDiagram + participant C as Create request + participant M as SandboxManager + participant P as RuntimeWarmPool + participant S as StorageProvider + participant B as Backend spawner + + C->>M: create(eligible decision) + M->>P: configure(prototype) + alt prototype is incompatible or capacity is disabled + P-->>M: rejected + M-->>C: existing create flow + else prototype is accepted + par Background work + P->>P: wake worker toward pool_size + P->>S: acquire independent storage + opt prefork enabled + P->>B: prepare and start backend + P->>B: check guest readiness when applicable + end + P->>P: persist Ready + and Request path + M->>P: acquire() + alt no ready slot + P-->>M: miss + M-->>C: existing create flow + else ready slot + P-->>M: lease + M->>P: persist Handoff(token) + M->>M: persist lifecycle owner(token) + M->>P: persist LifecycleOwned(token) + opt storage-only slot + M->>B: start backend + end + M-->>C: Running, start_path=warm + end + end + end +``` + +Construction is asynchronous. The request that first configures the prototype +does not wait for a slot to finish. The worker fills only while the number of +physical pool owners is below `pool_size`. + +Before calling provider allocation, the journal conservatively records that +storage may be owned. This permits release by ID if cancellation occurs after +the provider created an artifact but before it returned a complete slot. When +`prefork` is enabled, backend ownership is recorded as it moves from not +started, through starting, to running. A slot becomes `Ready` only after all +configured preparation succeeds. + +## Ownership States + +The persisted runtime journal uses these phases: + +```mermaid +stateDiagram-v2 + [*] --> Building + Building --> Ready: storage and optional backend prepared + Building --> PoolCleanup: build cannot complete + Ready --> Handoff: lease records token + Ready --> PoolCleanup: expiry or failed liveness check + Handoff --> LifecycleOwned: matching lifecycle owner is durable + Handoff --> PoolCleanup: lifecycle publication is absent + Handoff --> LifecycleCleanup: visible lifecycle owner begins cleanup + LifecycleOwned --> LifecycleCleanup: destroy or failed create cleanup + PoolCleanup --> [*]: backend, storage, and directory released + LifecycleCleanup --> [*]: runtime released; lifecycle state retained as Destroyed +``` + +The handoff token links one runtime journal to one lifecycle record. A phase +change is accepted only with the matching instance ID, backend, backend +ownership, and token. This prevents a stale cleanup path from reclaiming a +slot that a sandbox already owns. + +An in-memory lease has one synchronous drop owner: + +- before lifecycle publication, drop returns the slot to pool cleanup; +- after lifecycle state is retained, drop removes it from pool accounting and + leaves cleanup to the lifecycle owner; +- when publication cannot be classified, the pool retains an unresolved + handoff instead of choosing either owner. + +## Capacity Accounting + +The target counts physical resources, not only ready entries: + +```text +physical count = + ready + + building + + leased + + quarantined + + unresolved handoffs + + cleanup in progress +``` + +The worker starts another build only when this sum is below +`storage.pool_size`. A failed cleanup therefore cannot silently create excess +capacity. Once a handoff completes and the lease leaves pool accounting, the +active sandbox is governed by lifecycle ownership and no longer consumes the +runtime-slot target. + +## Failure Decisions + +| Failure point | Durable decision | Result | +| --- | --- | --- | +| Before or during storage allocation | `Building`, with conservative storage ownership before the call | Release by stable ID; retry cleanup if release fails | +| Backend prepare, start, or readiness | `Building` plus the latest backend ownership | Keep residual owners, move to pool cleanup, and retry | +| Ready slot expired or backend exited | Pool still owns the lease | Quarantine and clean it; try another ready slot | +| Lifecycle publication is definitely absent | `Handoff(token)` remains pool-authorized | Lease drop schedules pool cleanup | +| Lifecycle publication is visible but reports failure | Matching lifecycle record is retained | Finish lifecycle ownership, then run failed-create cleanup | +| Lifecycle publication is ambiguous | `Handoff(token)` is retained as unresolved | Keep it counted and require startup reconciliation | +| Pool or lifecycle cleanup stops partway through removal | The cleanup journal remains; a deletion proof also remains if directory removal began | Resume the same cleanup after restart | +| Pool cleanup attempt fails | Pool owner remains in its cleanup phase | Retry with backoff without returning the slot to ready | +| Lifecycle cleanup attempt fails | Lifecycle owner remains retryable | Report it; retry only through destroy, startup, or shutdown | + +Failure handling preserves the original error together with cleanup errors. +The caller never receives a successful warm create until lifecycle state is +`Running` and no create operation remains open. + +## Startup Reconciliation + +Runtime reconciliation runs before socket binding: + +```mermaid +flowchart TD + A["Load durable sandbox state"] --> B["Enumerate provider-owned slot IDs"] + B --> C["Read runtime journals and cleanup proofs"] + C --> D{"Ownership classification"} + D -- "Pool-owned or unclaimed" --> E["Clean backend, then storage and directory"] + D -- "Matching lifecycle owner" --> F["Preserve for lifecycle reconciliation"] + D -- "Incomplete cleanup" --> G["Resume recorded cleanup"] + D -- "Ambiguous or inconsistent" --> H["Stop daemon startup with an error"] + B -- "Inventory error" --> H + C -- "Read error" --> H + E -- "Cleanup error" --> H + G -- "Cleanup error" --> H + E -- "Success" --> J["Run sandbox lifecycle reconciliation"] + F --> J + G -- "Success" --> J + J --> K{"Any sandbox cleanup failure?"} + K -- "Yes" --> L["Retain and report each failure"] + K -- "No" --> I["Open API listeners"] + L --> I +``` + +The ready queue is process-local and is not restored. Valid unclaimed slots +are cleaned, while slots with a matching durable lifecycle owner are protected +for normal sandbox reconciliation. A runtime inventory, journal read, or +runtime cleanup error stops startup. Once runtime ownership is consistent, +ordinary sandbox reconciliation runs; one sandbox cleanup failure is retained +and reported without preventing listener startup. Unknown entries, mismatched +tokens, unexpected aliases, or conflicting ownership evidence also stop +startup rather than selecting an owner by guesswork. + +Recovery uses the backend recorded in the journal and the provider configured +for the same owned directories. Operators must keep those roots and provider +selection stable across restart. + +## Shutdown + +Shutdown first stops accepting work, cancels readiness waits, and drains +accepted connections. One shared deadline then covers concurrent lifecycle +cleanup and runtime-pool shutdown. The pool joins its single worker and cleans +pool-owned resources, while sandbox cleanup runs through per-sandbox operation +locks and lifecycle ownership. + +If the shutdown future itself is cancelled, the worker handle is retained so a +later shutdown attempt can still join it. A timeout stops new construction but +does not rewrite unresolved ownership as success. + +## Concurrency Rules + +- One maintenance task serializes build and cleanup actions. +- The in-memory state lock protects queues and counters but is not held across + provider or backend calls. +- One operation lock serializes create, destroy, and guest work for a sandbox. +- Runtime handoff is published before lifecycle mutation continues. +- Shutdown uses a fixed lock order and a shared deadline so worker join and + resource cleanup cannot wait indefinitely on one another. + +## Public Surface + +The feature changes `POST /v1/sandboxes` and its `/v1/instances` alias: +compatible requests may return a top-level `start_path` value of `"warm"` and +a nested `instance.runtime_location` value of `"warm-pool"`. `start_path` +remains a generic classification; `runtime_location` identifies this ownership +path. Configuration is the only public control surface for runtime-slot +capacity in this design. + +The following existing surfaces are separate: + +- `/v1/pools` exposes the separate lifecycle recycling-pool management + contract, but public reset still returns `501` and no production path returns + a used sandbox to that pool; +- the health response's `storage_pool` object reports provider storage-pool + status; +- neither surface reports, drains, or refills background runtime slots. + +## Test Mapping + +| Contract | Representative coverage | +| --- | --- | +| Storage-only and prefork claims complete create and destroy | `non_prefork_runtime_claim_completes_create_and_destroy`, `prefork_runtime_claim_completes_create_and_destroy` | +| Effective daemon TTL is visible when policy TTL is omitted | `omitted_policy_ttl_is_resolved_in_create_response` | +| All owned and in-flight states count toward the target | `every_owned_state_consumes_physical_capacity` | +| Failed allocation or prefork work retains cleanup ownership | `residual_acquire_failure_remains_owned_until_cleanup`, `residual_prefork_failure_remains_owned_until_cleanup` | +| Handoff ambiguity remains counted until reconciliation | `ambiguous_lifecycle_publish_remains_counted_until_restart`, `unresolved_handoff_counts_capacity_and_blocks_shutdown` | +| Startup protects lifecycle owners and cleans pool owners | `reconcile_protects_durable_lifecycle_owner`, `reconcile_releases_provider_only_slot_by_id` | +| Interrupted cleanup resumes from durable evidence | `reconcile_finishes_deletion_when_only_the_proof_remains`, `reconcile_resumes_lifecycle_tombstone_after_journal_unlink` | +| Shutdown joins the worker and preserves cancellation recovery | `shutdown_joins_worker_and_shares_one_deadline`, `cancelled_shutdown_retains_worker_for_a_joining_retry` | diff --git a/src/blaze/examples/policies/agent-rl.toml b/src/blaze/examples/policies/agent-rl.toml index deaa5be3fd..f6cbd03591 100644 --- a/src/blaze/examples/policies/agent-rl.toml +++ b/src/blaze/examples/policies/agent-rl.toml @@ -54,27 +54,32 @@ templates = [] fallback_on_missing_hook = "degrade" # --------------------------------------------------------------------------- -# Warm-pool configuration. Enables pre-created instances for fast cold-start. +# Warm-path policy metadata. Background slots also require a non-zero +# [storage].pool_size in config.toml. # --------------------------------------------------------------------------- [pool] -# Whether the warm pool is enabled for this policy. Default: false. +# Whether this policy is eligible for warm paths. Background runtime slots +# require this setting. Default: false. enabled = true -# Minimum number of warm instances to maintain. Default: 0. +# Reserved lifecycle recycling-pool schema metadata. The runtime-slot worker +# does not consume min/target/max, and /v1/pools does not apply them from policy. +# Reserved minimum. Default: 0. min = 4 -# Target pool size the autoscaler aims for. Default: 0. +# Reserved target. Default: 0. target = 16 -# Maximum pool size (hard cap). Default: 0. +# Reserved maximum. Default: 0. max = 64 -# Time-to-live for idle warm instances before the GC reclaims them. -# Overrides [pool].default_warm_ttl from config.toml for this policy. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". +# Ready-slot time-to-live override for this policy. If omitted, the daemon uses +# [pool].default_warm_ttl from config.toml (whose default is "30m"). +# Format: duration string (e.g. "30m", "1h"). warm_ttl = "30m" -# How the pool resets an instance after use so it can be returned to warm. +# Reserved lifecycle reset metadata. The public reset/return workflow is not +# connected; background slots are not returned to ready after destroy. # Options: "mm-template" (memory-template rollback, requires kernel hook), # "overlayfs-rollback" (filesystem-level rollback), # "full-recreate" (destroy and recreate from scratch). diff --git a/src/blaze/examples/policies/agent-tool.toml b/src/blaze/examples/policies/agent-tool.toml index ed6df07571..283cfe7900 100644 --- a/src/blaze/examples/policies/agent-tool.toml +++ b/src/blaze/examples/policies/agent-tool.toml @@ -47,26 +47,32 @@ templates = [] fallback_on_missing_hook = "fail" # --------------------------------------------------------------------------- -# Warm-pool configuration. +# Warm-path policy metadata. Background slots also require a non-zero +# [storage].pool_size in config.toml. # --------------------------------------------------------------------------- [pool] -# Whether the warm pool is enabled for this policy. Default: false. +# Whether this policy is eligible for warm paths. Background runtime slots +# require this setting. Default: false. enabled = true -# Minimum number of warm instances to maintain. Default: 0. +# Reserved lifecycle recycling-pool schema metadata. The runtime-slot worker +# does not consume min/target/max, and /v1/pools does not apply them from policy. +# Reserved minimum. Default: 0. min = 2 -# Target pool size the autoscaler aims for. Default: 0. +# Reserved target. Default: 0. target = 8 -# Maximum pool size (hard cap). Default: 0. +# Reserved maximum. Default: 0. max = 32 -# Time-to-live for idle warm instances before the GC reclaims them. -# Format: duration string (e.g. "30m", "1h"). Default: "30m". +# Ready-slot time-to-live override for this policy. If omitted, the daemon uses +# [pool].default_warm_ttl from config.toml (whose default is "30m"). +# Format: duration string (e.g. "30m", "1h"). warm_ttl = "30m" -# How the pool resets an instance after use. +# Reserved lifecycle reset metadata. The public reset/return workflow is not +# connected; background slots are not returned to ready after destroy. # Options: "mm-template", "overlayfs-rollback", "full-recreate". # Default: "mm-template". reset_mode = "full-recreate" From 1b45971979b209d1991ba741d0a15a71d62afdb2 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 08:43:37 +0800 Subject: [PATCH 25/40] feat(blaze): define checkpoint records The daemon had no stable record for checkpoint identity, parentage, runtime provenance, or artifact integrity. Backend snapshot requests also had no shared full-snapshot vocabulary. Add canonical checkpoint IDs, manifests, API views, artifact-set validation, and full snapshot request/result DTOs. This enables capture code in this PR to reject malformed catalog data before publication or listing; it does not expose a runtime operation by itself. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/backend.rs | 39 ++ src/blaze/crates/blaze-core/src/checkpoint.rs | 443 ++++++++++++++++++ src/blaze/crates/blaze-core/src/lib.rs | 2 + 3 files changed, 484 insertions(+) create mode 100644 src/blaze/crates/blaze-core/src/checkpoint.rs diff --git a/src/blaze/crates/blaze-core/src/backend.rs b/src/blaze/crates/blaze-core/src/backend.rs index e213e24397..f869f6d5ce 100644 --- a/src/blaze/crates/blaze-core/src/backend.rs +++ b/src/blaze/crates/blaze-core/src/backend.rs @@ -101,6 +101,37 @@ pub struct SpawnRequest { pub vm: Option, } +/// Snapshot flavor requested from a backend. +/// +/// The file provider currently requires self-contained artifacts, so only +/// full snapshots are exposed until a restore-independent delta format exists. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum SnapshotKind { + /// Self-contained VM and memory snapshot. + Full, +} + +/// Paths and semantics for one snapshot operation. +#[derive(Debug, Clone)] +pub struct SnapshotRequest { + /// Destination for VM state. + pub snapshot_path: PathBuf, + /// Destination for guest memory. + pub mem_path: PathBuf, + /// Snapshot flavor. + pub kind: SnapshotKind, +} + +/// Backend-reported snapshot artifacts. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SnapshotResult { + /// Written VM-state path. + pub snapshot_path: PathBuf, + /// Written memory path. + pub mem_path: PathBuf, +} + /// Probed availability of a single backend on this host. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct BackendStatus { @@ -205,4 +236,12 @@ mod tests { let err = select_backend(&priority, &available).expect_err("must fail"); assert!(matches!(err, BlazeError::BackendUnavailable { .. })); } + + #[test] + fn snapshot_kind_serializes_as_a_stable_lowercase_value() { + assert_eq!( + serde_json::to_value(SnapshotKind::Full).expect("snapshot kind"), + serde_json::json!("full") + ); + } } diff --git a/src/blaze/crates/blaze-core/src/checkpoint.rs b/src/blaze/crates/blaze-core/src/checkpoint.rs new file mode 100644 index 0000000000..e7ccc4d01a --- /dev/null +++ b/src/blaze/crates/blaze-core/src/checkpoint.rs @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Pure checkpoint records and manifest validation. +//! +//! This module deliberately contains no filesystem or path handling. The +//! daemon owns checkpoint persistence, hashing, publication, and cleanup. + +use std::collections::HashSet; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +use crate::backend::{BackendKind, SnapshotKind}; + +/// Current on-disk checkpoint metadata format. +pub const CHECKPOINT_FORMAT_VERSION: u32 = 1; + +/// Self-contained artifacts required for every committed checkpoint. +pub const REQUIRED_ARTIFACTS: [&str; 3] = ["vmstate.snap", "memory.snap", "rootfs.snap"]; + +/// One content digest recorded in a checkpoint manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointArtifact { + /// File name relative to the checkpoint directory. + pub name: String, + /// Logical file size in bytes. + pub size_bytes: u64, + /// Lowercase SHA-256 digest. + pub sha256: String, +} + +/// Durable checkpoint identity and integrity manifest. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointMetadata { + /// Metadata schema version. + pub format_version: u32, + /// Stable `ckpt-` identifier. + pub id: String, + /// Previous checkpoint on this branch. + #[serde(default)] + pub parent: Option, + /// Sandbox that owns the checkpoint. + pub sandbox_id: Uuid, + /// Policy that selected the captured runtime. + pub policy_name: String, + /// Image identity selected by the policy. + pub image_digest: String, + /// Backend that produced the runtime artifacts. + pub backend: BackendKind, + /// Backend version captured by the daemon, when available. + #[serde(default)] + pub backend_version: Option, + /// UTC publication time. + pub created_at: DateTime, + /// Backend snapshot semantics. + pub snapshot_kind: SnapshotKind, + /// Integrity records for all required artifacts. + pub artifacts: Vec, +} + +/// Read-only API view of a checkpoint. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct CheckpointInfo { + /// Checkpoint identifier. + pub id: String, + /// Parent checkpoint. + pub parent: Option, + /// Publication time. + pub created_at: DateTime, + /// Sum of logical artifact sizes. + pub size_bytes: u64, + /// Whether this checkpoint is the current HEAD. + pub is_head: bool, + /// Whether this checkpoint is reachable from HEAD. + pub on_head_chain: bool, +} + +/// Values supplied by the daemon when publishing a populated stage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CommitCheckpoint { + /// Parent checkpoint, if the sandbox already has a HEAD. + pub parent: Option, + /// Policy that selected the captured runtime. + pub policy_name: String, + /// Image identity selected by the policy. + pub image_digest: String, + /// Backend that produced the artifacts. + pub backend: BackendKind, + /// Backend version captured by the caller, when available. + pub backend_version: Option, + /// Backend snapshot semantics. + pub snapshot_kind: SnapshotKind, +} + +/// Pure validation failure for a checkpoint identifier or manifest. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CheckpointValidationError { + /// A checkpoint identifier is not canonical `ckpt-`. + #[error("invalid checkpoint identifier {checkpoint_id:?}: {reason}")] + InvalidIdentifier { + checkpoint_id: String, + reason: String, + }, + + /// A manifest uses a schema version this daemon cannot interpret. + #[error("checkpoint {checkpoint_id} uses unsupported format {actual}; expected {expected}")] + UnsupportedFormat { + checkpoint_id: String, + actual: u32, + expected: u32, + }, + + /// The durable identity does not agree with the catalog location. + #[error("checkpoint manifest identity mismatch: {reason}")] + IdentityMismatch { reason: String }, + + /// A manifest field needed to identify or reproduce a capture is empty. + #[error("checkpoint {checkpoint_id} has invalid {field}: {reason}")] + InvalidField { + checkpoint_id: String, + field: &'static str, + reason: String, + }, + + /// The artifact manifest is incomplete, duplicated, or malformed. + #[error("checkpoint {checkpoint_id} has an invalid artifact manifest: {reason}")] + InvalidArtifacts { + checkpoint_id: String, + reason: String, + }, + + /// A requested artifact name is outside the frozen format. + #[error("artifact name {name:?} is not part of the checkpoint format")] + InvalidArtifactName { name: String }, +} + +/// Validate a canonical `ckpt-` identifier. +pub fn validate_checkpoint_id(checkpoint_id: &str) -> Result { + let raw = checkpoint_id + .strip_prefix("ckpt-") + .ok_or_else(|| invalid_identifier(checkpoint_id, "missing ckpt- prefix"))?; + let uuid = Uuid::parse_str(raw) + .map_err(|error| invalid_identifier(checkpoint_id, error.to_string()))?; + if checkpoint_id != format!("ckpt-{uuid}") { + return Err(invalid_identifier( + checkpoint_id, + "identifier is not in canonical hyphenated lowercase form", + )); + } + Ok(uuid) +} + +/// Validate a name before the daemon resolves it inside a checkpoint stage. +pub fn validate_artifact_name(name: &str) -> Result<(), CheckpointValidationError> { + if REQUIRED_ARTIFACTS.contains(&name) { + Ok(()) + } else { + Err(CheckpointValidationError::InvalidArtifactName { + name: name.to_string(), + }) + } +} + +/// Validate daemon-supplied values before constructing a durable manifest. +pub fn validate_commit_checkpoint( + checkpoint_id: &str, + input: &CommitCheckpoint, +) -> Result<(), CheckpointValidationError> { + validate_checkpoint_id(checkpoint_id)?; + if let Some(parent) = &input.parent { + validate_checkpoint_id(parent)?; + if parent == checkpoint_id { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "parent", + reason: "a checkpoint cannot be its own parent".to_string(), + }); + } + } + validate_runtime_identity( + checkpoint_id, + &input.policy_name, + &input.image_digest, + input.backend_version.as_deref(), + ) +} + +/// Validate a parsed manifest against the catalog location that contained it. +/// +/// Artifact content hashes are intentionally not checked here: reading files +/// belongs to the daemon. This function only validates the pure record. +pub fn validate_checkpoint_manifest( + metadata: &CheckpointMetadata, + expected_sandbox_id: Uuid, + expected_checkpoint_id: &str, +) -> Result<(), CheckpointValidationError> { + validate_checkpoint_id(expected_checkpoint_id)?; + validate_checkpoint_id(&metadata.id)?; + if metadata.format_version != CHECKPOINT_FORMAT_VERSION { + return Err(CheckpointValidationError::UnsupportedFormat { + checkpoint_id: metadata.id.clone(), + actual: metadata.format_version, + expected: CHECKPOINT_FORMAT_VERSION, + }); + } + if metadata.id != expected_checkpoint_id { + return Err(CheckpointValidationError::IdentityMismatch { + reason: format!( + "manifest id {:?} does not match catalog id {expected_checkpoint_id:?}", + metadata.id + ), + }); + } + if metadata.sandbox_id != expected_sandbox_id { + return Err(CheckpointValidationError::IdentityMismatch { + reason: format!( + "manifest sandbox {} does not match catalog sandbox {expected_sandbox_id}", + metadata.sandbox_id + ), + }); + } + if let Some(parent) = &metadata.parent { + validate_checkpoint_id(parent)?; + if parent == &metadata.id { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: metadata.id.clone(), + field: "parent", + reason: "a checkpoint cannot be its own parent".to_string(), + }); + } + } + validate_runtime_identity( + &metadata.id, + &metadata.policy_name, + &metadata.image_digest, + metadata.backend_version.as_deref(), + )?; + validate_artifact_manifest(&metadata.id, &metadata.artifacts) +} + +fn validate_runtime_identity( + checkpoint_id: &str, + policy_name: &str, + image_digest: &str, + backend_version: Option<&str>, +) -> Result<(), CheckpointValidationError> { + if policy_name.trim().is_empty() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "policy_name", + reason: "value is empty".to_string(), + }); + } + if image_digest.trim().is_empty() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "image_digest", + reason: "value is empty".to_string(), + }); + } + if backend_version.is_some_and(|version| version.trim().is_empty()) { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "backend_version", + reason: "present version is empty".to_string(), + }); + } + Ok(()) +} + +fn validate_artifact_manifest( + checkpoint_id: &str, + artifacts: &[CheckpointArtifact], +) -> Result<(), CheckpointValidationError> { + if artifacts.len() != REQUIRED_ARTIFACTS.len() { + return Err(invalid_artifacts( + checkpoint_id, + format!( + "expected {} artifacts, found {}", + REQUIRED_ARTIFACTS.len(), + artifacts.len() + ), + )); + } + + let mut names = HashSet::with_capacity(artifacts.len()); + for artifact in artifacts { + validate_artifact_name(&artifact.name).map_err(|_| { + invalid_artifacts( + checkpoint_id, + format!("unexpected artifact {:?}", artifact.name), + ) + })?; + if !names.insert(artifact.name.as_str()) { + return Err(invalid_artifacts( + checkpoint_id, + format!("duplicate artifact {:?}", artifact.name), + )); + } + if artifact.sha256.len() != 64 + || !artifact + .sha256 + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(invalid_artifacts( + checkpoint_id, + format!("artifact {:?} has an invalid SHA-256 digest", artifact.name), + )); + } + } + if REQUIRED_ARTIFACTS + .iter() + .any(|required| !names.contains(required)) + { + return Err(invalid_artifacts( + checkpoint_id, + "one or more required artifacts are missing", + )); + } + Ok(()) +} + +fn invalid_identifier(checkpoint_id: &str, reason: impl Into) -> CheckpointValidationError { + CheckpointValidationError::InvalidIdentifier { + checkpoint_id: checkpoint_id.to_string(), + reason: reason.into(), + } +} + +fn invalid_artifacts(checkpoint_id: &str, reason: impl Into) -> CheckpointValidationError { + CheckpointValidationError::InvalidArtifacts { + checkpoint_id: checkpoint_id.to_string(), + reason: reason.into(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn artifact(name: &str, fill: char) -> CheckpointArtifact { + CheckpointArtifact { + name: name.to_string(), + size_bytes: 10, + sha256: std::iter::repeat_n(fill, 64).collect(), + } + } + + fn metadata() -> CheckpointMetadata { + let sandbox_id = Uuid::new_v4(); + CheckpointMetadata { + format_version: CHECKPOINT_FORMAT_VERSION, + id: format!("ckpt-{}", Uuid::new_v4()), + parent: None, + sandbox_id, + policy_name: "default".to_string(), + image_digest: "sha256:image".to_string(), + backend: BackendKind::Mock, + backend_version: Some("mock-v1".to_string()), + created_at: Utc::now(), + snapshot_kind: SnapshotKind::Full, + artifacts: vec![ + artifact("vmstate.snap", 'a'), + artifact("memory.snap", 'b'), + artifact("rootfs.snap", 'c'), + ], + } + } + + #[test] + fn canonical_identifier_round_trips() { + let uuid = Uuid::new_v4(); + assert_eq!( + validate_checkpoint_id(&format!("ckpt-{uuid}")).expect("valid identifier"), + uuid + ); + } + + #[test] + fn noncanonical_identifier_is_rejected() { + let uuid = Uuid::new_v4(); + assert!( + validate_checkpoint_id(&format!("ckpt-{}", uuid.to_string().to_uppercase())).is_err() + ); + assert!(validate_checkpoint_id(&format!("ckpt-{}", uuid.simple())).is_err()); + assert!(validate_checkpoint_id("../checkpoint").is_err()); + } + + #[test] + fn valid_manifest_passes_pure_validation() { + let metadata = metadata(); + validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect("valid manifest"); + } + + #[test] + fn manifest_identity_must_match_catalog_location() { + let metadata = metadata(); + let error = validate_checkpoint_manifest(&metadata, Uuid::new_v4(), &metadata.id) + .expect_err("sandbox mismatch must fail"); + assert!(matches!( + error, + CheckpointValidationError::IdentityMismatch { .. } + )); + } + + #[test] + fn manifest_requires_the_exact_artifact_set() { + let mut metadata = metadata(); + metadata.artifacts[2].name = "memory.snap".to_string(); + let error = validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect_err("duplicate artifact must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidArtifacts { .. } + )); + } + + #[test] + fn manifest_rejects_noncanonical_digest() { + let mut metadata = metadata(); + metadata.artifacts[0].sha256 = "A".repeat(64); + assert!( + validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id).is_err() + ); + } + + #[test] + fn commit_input_rejects_self_parent_and_empty_identity() { + let id = format!("ckpt-{}", Uuid::new_v4()); + let input = CommitCheckpoint { + parent: Some(id.clone()), + policy_name: String::new(), + image_digest: String::new(), + backend: BackendKind::Mock, + backend_version: None, + snapshot_kind: SnapshotKind::Full, + }; + assert!(validate_commit_checkpoint(&id, &input).is_err()); + } +} diff --git a/src/blaze/crates/blaze-core/src/lib.rs b/src/blaze/crates/blaze-core/src/lib.rs index 9ee302e5da..cd1e1908b6 100644 --- a/src/blaze/crates/blaze-core/src/lib.rs +++ b/src/blaze/crates/blaze-core/src/lib.rs @@ -9,6 +9,7 @@ //! - [`config`]: daemon TOML configuration //! - [`policy`]: workload class + policy file schema //! - [`backend`]: backend kinds + selection / fallback +//! - [`checkpoint`]: pure checkpoint records and manifest validation //! - [`guest_protocol`]: guest-agent wire DTOs //! - [`lifecycle`]: sandbox state machine + JSON persistence //! - [`pool`]: warm-pool key/stat/manager @@ -17,6 +18,7 @@ //! - [`error`]: unified [`BlazeError`] error enum pub mod backend; +pub mod checkpoint; pub mod config; pub mod error; pub mod guest_protocol; From 1c609d828b921e0925fcd6ed6a1d2c3499d5ca63 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 08:45:50 +0800 Subject: [PATCH 26/40] feat(blaze): journal checkpoint progress Lifecycle journals previously recorded only an operation kind and start time. After an interrupted capture, persisted state could not distinguish an unpublished stage from a committed checkpoint or a changed HEAD. Record checkpoint identity, monotonic durable phases, and the last completed capture, and allow a checkpointed sandbox to return to running. This exposes the latest boundary confirmed in persisted state while catalog listing reports what is actually visible; it does not add a provider or API entry point by itself. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/lifecycle.rs | 165 +++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index c270e546d8..735bc7db66 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -50,6 +50,8 @@ impl SandboxState { pub enum OperationKind { /// Sandbox creation is acquiring resources or starting a backend. Create, + /// A point-in-time checkpoint is being captured and published. + Checkpoint, /// Runtime resources are being destroyed. Destroy, } @@ -58,6 +60,7 @@ impl OperationKind { pub const fn as_str(&self) -> &'static str { match self { OperationKind::Create => "create", + OperationKind::Checkpoint => "checkpoint", OperationKind::Destroy => "destroy", } } @@ -69,6 +72,44 @@ impl std::fmt::Display for OperationKind { } } +/// Durable boundary reached by a multi-step lifecycle operation. +/// +/// The journal keeps this separate from [`SandboxState`]: state describes +/// externally visible runtime availability, while the phase identifies which +/// checkpoint resources may already have been published after interruption. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum OperationPhase { + /// A staging directory exists, but the backend has not been paused. + CheckpointPreparing, + /// The backend is paused while snapshot artifacts are being written. + CheckpointPaused, + /// A complete checkpoint directory is visible, but HEAD is unchanged. + CheckpointPublished, + /// HEAD references the checkpoint; runtime resume is not yet committed. + CheckpointHeadUpdated, +} + +impl OperationPhase { + const fn as_str(self) -> &'static str { + match self { + OperationPhase::CheckpointPreparing => "checkpoint-preparing", + OperationPhase::CheckpointPaused => "checkpoint-paused", + OperationPhase::CheckpointPublished => "checkpoint-published", + OperationPhase::CheckpointHeadUpdated => "checkpoint-head-updated", + } + } + + const fn rank(self) -> u8 { + match self { + OperationPhase::CheckpointPreparing => 0, + OperationPhase::CheckpointPaused => 1, + OperationPhase::CheckpointPublished => 2, + OperationPhase::CheckpointHeadUpdated => 3, + } + } +} + /// Durable journal entry for one active lifecycle operation. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OperationJournal { @@ -76,6 +117,12 @@ pub struct OperationJournal { pub kind: OperationKind, /// UTC time at which the operation became externally visible. pub started_at: DateTime, + /// Checkpoint selected by this operation, when applicable. + #[serde(default)] + pub checkpoint_id: Option, + /// Last durably committed operation boundary. + #[serde(default)] + pub phase: Option, } impl std::fmt::Display for SandboxState { @@ -125,6 +172,9 @@ pub struct SandboxInstance { /// Active multi-step operation, if any. #[serde(default)] pub operation: Option, + /// Last checkpoint whose capture completed and returned the sandbox to running. + #[serde(default)] + pub last_checkpoint: Option, } impl SandboxInstance { @@ -151,6 +201,7 @@ impl SandboxInstance { policy_name, backend_ownership: BackendOwnership::NotStarted, operation: None, + last_checkpoint: None, } } @@ -167,11 +218,54 @@ impl SandboxInstance { self.operation = Some(OperationJournal { kind, started_at: Utc::now(), + checkpoint_id: None, + phase: None, }); self.updated_at = Utc::now(); Ok(()) } + /// Record checkpoint intent before pausing the backend. + pub fn begin_checkpoint_operation(&mut self, checkpoint_id: String) -> Result<()> { + self.begin_operation(OperationKind::Checkpoint)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.checkpoint_id = Some(checkpoint_id); + operation.phase = Some(OperationPhase::CheckpointPreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active checkpoint journal without replacing its identity. + pub fn advance_checkpoint_phase(&mut self, phase: OperationPhase) -> Result<()> { + let operation = self + .operation + .as_mut() + .ok_or_else(|| BlazeError::OperationInProgress { + active: "none".to_string(), + requested: OperationKind::Checkpoint.to_string(), + })?; + if operation.kind != OperationKind::Checkpoint { + return Err(BlazeError::OperationInProgress { + active: operation.kind.to_string(), + requested: OperationKind::Checkpoint.to_string(), + }); + } + if let Some(current) = operation.phase + && phase.rank() < current.rank() + { + return Err(BlazeError::InvalidStateTransition { + from: current.as_str().to_string(), + to: phase.as_str().to_string(), + }); + } + operation.phase = Some(phase); + self.updated_at = Utc::now(); + Ok(()) + } + /// Transfer an interrupted lifecycle operation to destroy recovery. /// /// Cleanup is the only operation allowed to supersede an unfinished @@ -183,6 +277,8 @@ impl SandboxInstance { self.operation = Some(OperationJournal { kind: OperationKind::Destroy, started_at: Utc::now(), + checkpoint_id: None, + phase: None, }); self.updated_at = Utc::now(); } @@ -334,6 +430,7 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { (Running, Reset) => true, (Paused, Checkpointed) => true, (Paused, Running) => true, // resume + (Checkpointed, Running) => true, (Reset, Warm) => true, (Warm, Creating) => true, // pool reuse / warm path _ => false, @@ -364,6 +461,7 @@ mod tests { SandboxState::Running, SandboxState::Paused, SandboxState::Checkpointed, + SandboxState::Running, SandboxState::Destroyed, ] { inst.transition(target).expect("legal transition"); @@ -576,6 +674,7 @@ mod tests { }); let loaded: SandboxInstance = serde_json::from_value(value).expect("legacy state"); assert!(loaded.operation.is_none()); + assert!(loaded.last_checkpoint.is_none()); assert_eq!(loaded.backend_ownership, BackendOwnership::Unknown); } @@ -616,4 +715,70 @@ mod tests { )); assert_eq!(instance.operation, Some(journal)); } + + #[test] + fn checkpoint_journal_preserves_identity_and_phase() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect("begin checkpoint"); + instance + .advance_checkpoint_phase(OperationPhase::CheckpointPublished) + .expect("advance checkpoint"); + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + let journal = loaded.operation.expect("checkpoint journal"); + assert_eq!(journal.kind, OperationKind::Checkpoint); + assert_eq!( + journal.checkpoint_id.as_deref(), + Some("ckpt-00000000-0000-0000-0000-000000000001") + ); + assert_eq!(journal.phase, Some(OperationPhase::CheckpointPublished)); + } + + #[test] + fn checkpoint_journal_rejects_phase_regression() { + let mut instance = fresh(); + instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect("begin checkpoint"); + instance + .advance_checkpoint_phase(OperationPhase::CheckpointPublished) + .expect("advance checkpoint"); + + let error = instance + .advance_checkpoint_phase(OperationPhase::CheckpointPaused) + .expect_err("checkpoint phase must remain a durable lower bound"); + + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!( + instance + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + } + + #[test] + fn checkpoint_journal_cannot_replace_an_active_operation() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone(); + + let error = instance + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".into()) + .expect_err("checkpoint must not replace create"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "checkpoint" + )); + assert_eq!(instance.operation, journal); + } } From 6832c4535ff4fe95068790cc7ae14120cc02bbdc Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 08:46:02 +0800 Subject: [PATCH 27/40] feat(blaze): capture sandbox checkpoints Checkpoint requests previously stopped after lifecycle validation with 501. The daemon had no provider contract, durable catalog, or list operation for captured state. Add opt-in backend and storage capture contracts, self-contained file-provider copies, a mock backend implementation, verified catalog publication with HEAD tracking, and serialized capture/list API paths. Compensation resumes the runtime and removes unpublished work, while uncertain publication or resume outcomes retain ownership for recovery. This enables callers to capture and list independent checkpoints on supporting backends. Unsupported backends or storage providers fail before journal or catalog mutation. The contracts, first implementations, catalog coordinator, and API stay together because they form one production call path and none is independently usable here. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/AGENTS.md | 10 +- src/blaze/Cargo.lock | 72 + src/blaze/Cargo.toml | 1 + src/blaze/README.md | 45 +- src/blaze/README_zh.md | 36 +- src/blaze/crates/blaze-core/src/storage.rs | 18 +- src/blaze/crates/blazed/Cargo.toml | 1 + src/blaze/crates/blazed/src/api.rs | 740 ++++++++++- .../crates/blazed/src/checkpoint_store.rs | 1180 +++++++++++++++++ src/blaze/crates/blazed/src/file_provider.rs | 418 +++++- src/blaze/crates/blazed/src/main.rs | 1 + src/blaze/crates/blazed/src/sandbox.rs | 1 + .../crates/blazed/src/sandbox/checkpoint.rs | 442 ++++++ .../crates/blazed/src/sandbox/manager.rs | 36 +- src/blaze/crates/blazed/src/spawner.rs | 170 ++- 15 files changed, 3128 insertions(+), 43 deletions(-) create mode 100644 src/blaze/crates/blazed/src/checkpoint_store.rs create mode 100644 src/blaze/crates/blazed/src/sandbox/checkpoint.rs diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index 7580ddefe6..c93cca5ac0 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -30,10 +30,12 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. - **Lifecycle state machine**: The persisted model contains 9 states. Current - managed routes implement create, destroy, and RecoveryRequired cleanup. - Checkpoint and reset routes are reserved and return `501` without advancing - state. Transitions are enforced by `blaze_core::lifecycle`; do not bypass - them through direct field mutation. + managed routes implement create, destroy, checkpoint capture/list, and + RecoveryRequired cleanup. Checkpoint capture returns `501` without advancing + state when either the backend or storage provider lacks capture support. + Reset remains reserved and returns `501` without advancing state. + Transitions are enforced by `blaze_core::lifecycle`; do not bypass them + through direct field mutation. - **MockSpawner fallback**: When the configured backend binary is missing or fails `probe()`, the daemon auto-downgrades to `MockSpawner` with a warning. This keeps API/integration tests functional without a real backend. ## Adding a New Backend diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 297eb282a6..152ccdc5c6 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -143,6 +143,7 @@ dependencies = [ "libc", "serde", "serde_json", + "sha2", "tempfile", "thiserror", "tokio", @@ -153,6 +154,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bumpalo" version = "3.20.3" @@ -255,6 +265,35 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -346,6 +385,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.4.2" @@ -836,6 +885,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1138,6 +1198,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1174,6 +1240,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + [[package]] name = "want" version = "0.3.1" diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index a67d5f6820..1a23d8ad02 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -43,6 +43,7 @@ uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", default-features = false, features = ["std", "clock", "serde"] } semver = "1.0" base64 = "0.22" +sha2 = "0.10" # Platform libc = "0.2" diff --git a/src/blaze/README.md b/src/blaze/README.md index 69b4a13035..cbd66a0729 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -138,6 +138,8 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/sandboxes/{id}/exec` | Execute a guest command | | POST | `/v1/sandboxes/{id}/read` | Read a guest file | | POST | `/v1/sandboxes/{id}/write` | Replace a guest file | +| POST | `/v1/sandboxes/{id}/checkpoint` | Capture a full checkpoint when the backend and storage provider support it | +| GET | `/v1/sandboxes/{id}/checkpoints` | List committed checkpoints and HEAD reachability | | GET | `/v1/instances` | Alias for listing sandboxes | | POST | `/v1/instances` | Alias for creating a sandbox | | GET | `/v1/instances/{id}` | Alias for sandbox details | @@ -146,7 +148,8 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances/{id}/exec` | Compatible guest command action | | POST | `/v1/instances/{id}/read` | Compatible guest file read action | | POST | `/v1/instances/{id}/write` | Compatible guest file write action | -| POST | `/v1/instances/{id}/checkpoint` | Reserved; returns `501` until backend and storage capture is implemented | +| POST | `/v1/instances/{id}/checkpoint` | Compatible full-checkpoint action | +| GET | `/v1/instances/{id}/checkpoints` | Compatible checkpoint-list action | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | @@ -176,13 +179,39 @@ accepted connections. It then attempts bounded cleanup for every persisted record and retained backend owner. One cleanup failure does not skip the remaining sandboxes, and all unresolved records are reported. -The operation journal records the operation and start time, not completion of -each resource step. An interrupted create is cleaned up rather than resumed, -and an existing backend process is not adopted after restart. Failed recovery -does not run in a background retry loop. Checkpoint and reset validate that the -sandbox is running with no active lifecycle operation, then return `501` -without changing runtime or persisted state. Their backend and storage -operations are not implemented here. +Create and destroy journals record the operation and start time. Checkpoint +journals also record the generated checkpoint ID and the latest durable +boundary the daemon confirmed. Checkpoint listing separately reports which +catalog entries and HEAD update are actually visible after an interruption. +An interrupted create is cleaned up rather than resumed, and an existing +backend process is not adopted after restart. Failed recovery does not run in +a background retry loop. + +Checkpoint capture is available only when both the selected backend and the +configured storage provider report full-capture support. Otherwise the daemon +returns `501` before creating a journal, pausing the backend, or changing the +checkpoint catalog. A supported capture: + +1. pauses the backend and captures full VM and memory artifacts; +2. flushes the live storage slot and copies the full root filesystem; +3. publishes a verified checkpoint and advances HEAD; and +4. resumes the backend and confirms guest readiness before returning + `Running`. + +The file storage provider copies the complete root filesystem into each +checkpoint. This uses more capacity than a shared-base format, but each +checkpoint remains independent of later changes to the live slot. + +Failures detected before calling the catalog publication step resume the +backend and discard the incomplete stage. If publication or HEAD has an +uncertain outcome, or the backend cannot resume, the sandbox becomes +`RecoveryRequired` while runtime ownership and committed checkpoint data +remain available for explicit cleanup. Listing uses the same per-sandbox +operation lock as capture, guest operations, and destroy. Destroy removes +transaction scratch but preserves committed checkpoint history. + +Runtime reset remains reserved and returns `501` without changing runtime or +persisted state. ### Guest operations diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index df9d643e3e..bf34d8d52f 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -134,6 +134,8 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/sandboxes/{id}/exec` | 执行 guest 命令 | | POST | `/v1/sandboxes/{id}/read` | 读取 guest 文件 | | POST | `/v1/sandboxes/{id}/write` | 替换 guest 文件 | +| POST | `/v1/sandboxes/{id}/checkpoint` | 后端和存储 provider 支持时捕获完整 checkpoint | +| GET | `/v1/sandboxes/{id}/checkpoints` | 列出已提交的 checkpoint 及其 HEAD 可达性 | | GET | `/v1/instances` | 列出 sandbox 的兼容入口 | | POST | `/v1/instances` | 创建 sandbox 的兼容入口 | | GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | @@ -142,7 +144,8 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances/{id}/exec` | Guest 命令兼容入口 | | POST | `/v1/instances/{id}/read` | Guest 文件读取兼容入口 | | POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | -| POST | `/v1/instances/{id}/checkpoint` | 预留接口；后端和存储快照实现前返回 `501` | +| POST | `/v1/instances/{id}/checkpoint` | 捕获完整 checkpoint 的兼容入口 | +| GET | `/v1/instances/{id}/checkpoints` | 列出 checkpoint 的兼容入口 | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | @@ -169,11 +172,32 @@ daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失 记录和仍持有的后端资源执行有界清理。单条清理失败不会跳过其余 sandbox， 所有未完成记录都会汇总报告。 -操作记录只保存操作类型和开始时间，不记录每个资源步骤是否已经完成。中断的 -创建会被清理而不是从原位置继续，重启后也不会接管先前的后端进程。恢复失败 -后目前没有后台循环自动重试。checkpoint 和 reset 会先确认 sandbox 处于 -`Running` 且没有进行中的生命周期操作，然后返回 `501`；它们不会修改运行 -资源或持久化状态，其后端和存储操作尚未在这里实现。 +创建和销毁的操作记录会保存操作类型及开始时间。checkpoint 还会记录生成的 +checkpoint ID，以及 daemon 已确认的最近一次持久化边界。checkpoint 列表会 +另外报告中断后实际可见的 catalog 记录和 HEAD 更新。中断的创建会被清理而 +不是从原位置继续，重启后也不会接管先前的后端进程。恢复失败后目前没有后台 +循环自动重试。 + +只有所选后端和当前存储 provider 都声明支持完整捕获时，checkpoint 才可用。 +否则 daemon 会在创建操作记录、暂停后端或修改 checkpoint catalog 之前返回 +`501`。一次成功的捕获会： + +1. 暂停后端，并捕获完整的 VM 状态和内存； +2. 刷新当前存储 slot，并复制完整根文件系统； +3. 发布经过校验的 checkpoint，并推进 HEAD； +4. 恢复后端，确认 guest 已经就绪后再返回 `Running`。 + +file storage provider 会把完整根文件系统复制到每个 checkpoint。与共享 base +的格式相比，这会占用更多空间，但每个 checkpoint 都不依赖 live slot 后续的 +变化。 + +如果在调用 catalog 发布步骤之前发现失败，daemon 会恢复后端并删除未完成的 +stage。如果发布或 HEAD 的结果无法确定，或者后端无法恢复，sandbox 会进入 +`RecoveryRequired`；runtime ownership 和已经提交的 checkpoint 数据仍会 +保留，供后续显式清理。列出 checkpoint 与捕获、guest 操作及销毁共用同一个 +sandbox 操作锁。销毁会删除事务临时文件，但保留已经提交的 checkpoint 历史。 + +runtime reset 仍是预留接口，会返回 `501`，且不会修改 runtime 或持久化状态。 ### Guest 操作 diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index aa4961a6ca..8bc5c00d15 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -5,7 +5,7 @@ //! (warm pools, copy-on-write, content-addressable dedup) but present //! a uniform interface to the daemon layer. -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use async_trait::async_trait; use thiserror::Error; @@ -128,6 +128,22 @@ pub trait StorageProvider: Send + Sync { /// Flush dirty data to persistent storage (implementation may be no-op). async fn flush_dirty(&self, slot: &StorageSlot) -> Result<()>; + /// Report whether this provider can capture a self-contained checkpoint. + /// + /// The default is conservative so existing providers do not advertise a + /// data path they have not implemented. + fn supports_checkpoint_capture(&self) -> bool { + false + } + + /// Capture the slot's writable root filesystem at `target`. + async fn capture_checkpoint(&self, slot: &StorageSlot, target: &Path) -> Result<()> { + let _ = (slot, target); + Err(BlazeError::StorageError { + msg: "storage provider does not support checkpoint capture".to_string(), + }) + } + /// Query warm pool status. fn pool_status(&self) -> PoolStatus; diff --git a/src/blaze/crates/blazed/Cargo.toml b/src/blaze/crates/blazed/Cargo.toml index 67e1d58c4a..790a3c65e7 100644 --- a/src/blaze/crates/blazed/Cargo.toml +++ b/src/blaze/crates/blazed/Cargo.toml @@ -36,6 +36,7 @@ uuid = { workspace = true } chrono = { workspace = true } libc = { workspace = true } base64 = { workspace = true } +sha2 = { workspace = true } [dev-dependencies] tempfile = "3" diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index fe77d055c2..a7bb54fe02 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -118,7 +118,10 @@ async fn dispatch( ("POST", ["v1", "sandboxes", id, "write"]) | ("POST", ["v1", "instances", id, "write"]) => { write_instance_file(state, id, &body).await } - ("POST", ["v1", "instances", id, "checkpoint"]) => checkpoint(state, id).await, + ("POST", ["v1", "instances", id, "checkpoint"]) + | ("POST", ["v1", "sandboxes", id, "checkpoint"]) => checkpoint(state, id).await, + ("GET", ["v1", "instances", id, "checkpoints"]) + | ("GET", ["v1", "sandboxes", id, "checkpoints"]) => list_checkpoints(state, id).await, ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, ("DELETE", ["v1", "instances", id]) | ("DELETE", ["v1", "sandboxes", id]) @@ -302,14 +305,11 @@ async fn create_instance(state: &Arc, body: &[u8]) -> Result, id: &str) -> Result>> { let uuid = parse_uuid(id)?; - let _operation = state - .manager - .lock_quiescent_state(uuid, SandboxState::Running) - .await?; + json_ok(&state.manager.checkpoint(uuid).await?) +} - Err(BlazeDaemonError::UnsupportedOperation(format!( - "instance {uuid} cannot be checkpointed until its backend and storage state can be captured" - ))) +async fn list_checkpoints(state: &Arc, id: &str) -> Result>> { + json_ok(&state.manager.list_checkpoints(parse_uuid(id)?).await?) } async fn reset_instance(state: &Arc, id: &str) -> Result>> { @@ -848,6 +848,8 @@ mod tests { use blaze_core::backend::{BackendKind, SpawnRequest}; use blaze_core::config::DaemonConfig; use blaze_core::kernel::HookRegistry; + #[cfg(feature = "test-failpoints")] + use blaze_core::lifecycle::OperationPhase; use blaze_core::lifecycle::{BackendOwnership, OperationKind}; use blaze_core::policy::{ BackendConfigs, FallbackOnMissingHook, PolicyEngine, PolicyFile, PolicyHooks, PolicyMatch, @@ -951,7 +953,11 @@ mod tests { #[cfg(feature = "test-failpoints")] fn mock_state(temp: &tempfile::TempDir, pooled: bool) -> Arc { - let config = test_config(temp); + mock_state_from_config(test_config(temp), pooled) + } + + #[cfg(feature = "test-failpoints")] + fn mock_state_from_config(config: DaemonConfig, pooled: bool) -> Arc { let storage: Arc = Arc::new(FileStorageProvider::with_images( config.storage.images_dir.clone(), config.storage.instances_dir.clone(), @@ -978,6 +984,73 @@ mod tests { .expect("created json") } + async fn write_checkpoint_fixture(state: &Arc, id: &str) -> StorageSlot { + let slot = state.storage.reconstruct(id).await.expect("storage slot"); + tokio::fs::write(&slot.rootfs_path, b"checkpoint-rootfs") + .await + .expect("rootfs"); + slot + } + + #[cfg(feature = "test-failpoints")] + async fn cancel_checkpoint_at(state: &Arc, id: Uuid, failpoint: &'static str) { + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = + tokio::spawn( + async move { capture_hook.run(capture_state.manager.checkpoint(id)).await }, + ); + hook.wait_until_paused().await; + capture.abort(); + let cancelled = capture + .await + .expect_err("checkpoint task must be cancelled"); + assert!(cancelled.is_cancelled()); + } + + struct NoCheckpointStorage { + inner: FileStorageProvider, + } + + #[async_trait] + impl StorageProvider for NoCheckpointStorage { + async fn probe(&self) -> blaze_core::Result { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> blaze_core::Result<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> blaze_core::Result<()> { + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> blaze_core::Result { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> blaze_core::Result<()> { + self.inner.flush_dirty(slot).await + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> blaze_core::Result { + self.inner.drain_pool().await + } + } + async fn dispatched_json( state: &Arc, method: Method, @@ -1795,13 +1868,15 @@ mod tests { } #[tokio::test] - async fn checkpoint_rejects_state_only_snapshot() { + async fn checkpoint_rejects_unsupported_storage_without_mutation() { let temp = tempfile::tempdir().expect("temp"); let config = test_config(&temp); - let storage: Arc = Arc::new(FileStorageProvider::with_images( - config.storage.images_dir.clone(), - config.storage.instances_dir.clone(), - )); + let storage: Arc = Arc::new(NoCheckpointStorage { + inner: FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + ), + }); let state = build_test_state( config, test_policy(BackendKind::Mock, false), @@ -1826,13 +1901,650 @@ mod tests { state.instances.lock().expect("instances")[&uuid].state, SandboxState::Running ); + assert!( + state.instances.lock().expect("instances")[&uuid] + .operation + .is_none() + ); + assert_eq!( + std::fs::read(state_path).expect("persisted state"), + persisted_before + ); + assert!(!state.state_dir.join("checkpoints").join(id).exists()); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_rejects_unsupported_backend_without_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let kill_count = Arc::new(AtomicUsize::new(0)); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners( + BackendKind::Mock, + Arc::new(CountingSpawner { + kill_count: kill_count.clone(), + orphan_cleanup_count: Arc::new(AtomicUsize::new(0)), + }), + ), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let state_path = state.state_dir.join(id).join("state.json"); + let persisted_before = std::fs::read(&state_path).expect("persisted state"); + + let error = checkpoint(&state, id) + .await + .expect_err("checkpoint without backend capture must fail closed"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!(error.status_code(), 501); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); assert_eq!( std::fs::read(state_path).expect("persisted state"), persisted_before ); + assert!(!state.state_dir.join("checkpoints").join(id).exists()); + assert_eq!(kill_count.load(Ordering::Acquire), 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[tokio::test] + async fn checkpoint_routes_capture_and_list_live_state() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + + let (status, checkpoint) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoint"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + let checkpoint_id = checkpoint["id"].as_str().expect("checkpoint id"); + assert_eq!(checkpoint["snapshot_kind"], "full"); + assert_eq!(checkpoint["sandbox_id"], id); + let captured_rootfs = state + .state_dir + .join("checkpoints") + .join(id) + .join(checkpoint_id) + .join("rootfs.snap"); + assert_eq!( + tokio::fs::read(&captured_rootfs) + .await + .expect("captured rootfs"), + b"checkpoint-rootfs" + ); + + tokio::fs::write(&slot.rootfs_path, b"changed-after-checkpoint") + .await + .expect("mutate live rootfs"); + assert_eq!( + tokio::fs::read(&captured_rootfs) + .await + .expect("independent captured rootfs"), + b"checkpoint-rootfs" + ); + let (status, checkpoints) = dispatched_json( + &state, + Method::GET, + &format!("/v1/instances/{id}/checkpoints"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(checkpoints.as_array().expect("checkpoint list").len(), 1); + assert_eq!(checkpoints[0]["id"], checkpoint_id); + assert_eq!(checkpoints[0]["is_head"], true); + assert_eq!(checkpoints[0]["on_head_chain"], true); + + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!(lifecycle.last_checkpoint.as_deref(), Some(checkpoint_id)); assert!(state.manager.backend_owner(uuid).is_some()); } + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_snapshot_failure_resumes_and_clears_the_journal() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-snapshot"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("snapshot failure"); + + assert!(matches!( + error, + BlazeDaemonError::Core(BlazeError::BackendError { .. }) + )); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!( + SandboxInstance::load(&state.state_dir, uuid) + .expect("persisted lifecycle") + .operation, + None + ); + let checkpoint_dir = state.state_dir.join("checkpoints").join(id); + let staging = std::fs::read_dir(checkpoint_dir) + .expect("checkpoint directory") + .filter_map(std::result::Result::ok) + .filter(|entry| entry.file_name().to_string_lossy().starts_with(".ckpt-")) + .count(); + assert_eq!(staging, 0); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_prepublication_failure_discards_the_stage() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-publish"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("publication must fail before the store call"); + + assert!(matches!( + error, + BlazeDaemonError::Core(BlazeError::StorageError { .. }) + )); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint catalog") + .is_empty() + ); + assert!(state.manager.backend_owner(uuid).is_some()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_state_failures_retain_the_reached_durable_phase() { + for (failpoint, expected_phase, expected_head) in [ + ( + "checkpoint-published-state", + OperationPhase::CheckpointPublished, + false, + ), + ( + "checkpoint-head-state", + OperationPhase::CheckpointHeadUpdated, + true, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("state commit must fail"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(expected_phase) + ); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("published checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0].is_head, expected_head); + assert!(state.manager.backend_owner(uuid).is_some()); + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_store_boundary_failures_preserve_observable_catalog_truth() { + for (failpoint, expected_phase, expected_head) in [ + ( + "checkpoint-store-publish-after-rename", + OperationPhase::CheckpointPaused, + false, + ), + ( + "checkpoint-store-head-after-rename", + OperationPhase::CheckpointPublished, + true, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("durability boundary must report an uncertain result"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(expected_phase) + ); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("observable checkpoint catalog"); + assert_eq!(checkpoints.len(), 1); + assert_eq!(checkpoints[0].is_head, expected_head); + assert!(state.manager.backend_owner(uuid).is_some()); + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_resume_failure_keeps_head_and_runtime_ownership() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-resume"]); + + let error = hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("resume failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointHeadUpdated) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("committed checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert!(checkpoints[0].is_head); + + state.manager.destroy(uuid).await.expect("destroy retry"); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("durable checkpoint history") + .len(), + 1 + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn published_checkpoint_holds_the_operation_lock_until_head_commit() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted checkpoint journal"); + assert_eq!(persisted.state, SandboxState::Paused); + assert_eq!( + persisted.operation.and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + let list_state = state.clone(); + let mut list = tokio::spawn(async move { list_state.manager.list_checkpoints(uuid).await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut list) + .await + .is_err(), + "checkpoint listing must wait for a consistent catalog boundary" + ); + let destroy_state = state.clone(); + let mut destroy = tokio::spawn(async move { destroy_state.manager.destroy(uuid).await }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut destroy) + .await + .is_err(), + "destroy must wait for checkpoint ownership" + ); + + hook.release(); + capture + .await + .expect("capture task") + .expect("checkpoint capture"); + let checkpoints = list.await.expect("list task").expect("checkpoint list"); + assert_eq!(checkpoints.len(), 1); + assert!(checkpoints[0].is_head); + assert!(destroy.await.expect("destroy task").expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_published_checkpoint_is_destroyable() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + capture.abort(); + let _ = capture.await; + + let interrupted = state.manager.get(uuid).expect("interrupted lifecycle"); + assert_eq!(interrupted.state, SandboxState::Paused); + assert_eq!( + interrupted.operation.and_then(|journal| journal.phase), + Some(OperationPhase::CheckpointPublished) + ); + assert!( + !state + .state_dir + .join("checkpoints") + .join(&id) + .join("HEAD") + .exists() + ); + + state + .manager + .destroy(uuid) + .await + .expect("destroy interrupted capture"); + let checkpoints = state + .manager + .list_checkpoints(uuid) + .await + .expect("unreachable checkpoint"); + assert_eq!(checkpoints.len(), 1); + assert!(!checkpoints[0].is_head); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_checkpoint_phases_are_destroyable_before_and_after_restart() { + for (failpoint, expected_state, expected_phase, checkpoint_count) in [ + ( + "checkpoint-after-begin", + SandboxState::Running, + OperationPhase::CheckpointPreparing, + 0, + ), + ( + "checkpoint-after-pause", + SandboxState::Paused, + OperationPhase::CheckpointPaused, + 0, + ), + ( + "checkpoint-after-head", + SandboxState::Paused, + OperationPhase::CheckpointHeadUpdated, + 1, + ), + ] { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + cancel_checkpoint_at(&state, uuid, failpoint).await; + let interrupted = state.manager.get(uuid).expect("interrupted lifecycle"); + assert_eq!(interrupted.state, expected_state); + assert_eq!( + interrupted.operation.and_then(|journal| journal.phase), + Some(expected_phase) + ); + + state + .manager + .destroy(uuid) + .await + .expect("same-process destroy"); + let destroyed = state.manager.get(uuid).expect("destroyed lifecycle"); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint history") + .len(), + checkpoint_count + ); + + let restart_temp = tempfile::tempdir().expect("restart temp"); + let config = test_config(&restart_temp); + let restart_state = mock_state_from_config(config.clone(), false); + let created = created_json(&restart_state, &test_request()).await; + let restart_id = created["instance"]["id"] + .as_str() + .expect("restart id") + .to_string(); + let restart_uuid = Uuid::parse_str(&restart_id).expect("restart uuid"); + write_checkpoint_fixture(&restart_state, &restart_id).await; + cancel_checkpoint_at(&restart_state, restart_uuid, failpoint).await; + restart_state + .manager + .backend_owner(restart_uuid) + .expect("backend owner") + .kill() + .await + .expect("simulate daemon exit"); + drop(restart_state); + + let restarted = mock_state_from_config(config, false); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); + let destroyed = restarted + .manager + .get(restart_uuid) + .expect("reconciled lifecycle"); + assert_eq!(destroyed.state, SandboxState::Destroyed); + assert!(destroyed.operation.is_none()); + assert_eq!( + restarted + .manager + .list_checkpoints(restart_uuid) + .await + .expect("reconciled checkpoint history") + .len(), + checkpoint_count + ); + let checkpoint_dir = restarted.state_dir.join("checkpoints").join(&restart_id); + if checkpoint_dir.exists() { + assert!( + !std::fs::read_dir(checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with('.')) + ); + } + } + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn guest_operations_wait_for_checkpoint_publication() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + state + .manager + .write_file(uuid, "/tmp/existing".into(), b"before") + .await + .expect("seed guest file"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-after-publish-before-head"]); + let capture_state = state.clone(); + let capture_hook = hook.clone(); + let capture = tokio::spawn(async move { + capture_hook + .run(capture_state.manager.checkpoint(uuid)) + .await + }); + hook.wait_until_paused().await; + + let exec_state = state.clone(); + let mut exec = tokio::spawn(async move { + exec_state + .manager + .exec(uuid, "printf locked".into(), None, None, 5) + .await + }); + let read_state = state.clone(); + let mut read = tokio::spawn(async move { + read_state + .manager + .read_file(uuid, "/tmp/existing".into()) + .await + }); + let write_state = state.clone(); + let mut write = tokio::spawn(async move { + write_state + .manager + .write_file(uuid, "/tmp/after".into(), b"after") + .await + }); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut exec) + .await + .is_err(), + "guest exec must wait for checkpoint ownership" + ); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut read) + .await + .is_err(), + "guest read must wait for checkpoint ownership" + ); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut write) + .await + .is_err(), + "guest write must wait for checkpoint ownership" + ); + + hook.release(); + capture + .await + .expect("capture task") + .expect("checkpoint capture"); + assert_eq!( + exec.await.expect("exec task").expect("guest exec").stdout, + b"printf locked" + ); + assert_eq!( + read.await.expect("read task").expect("guest read"), + b"before" + ); + write.await.expect("write task").expect("guest write"); + } + #[tokio::test] async fn reset_rejects_state_only_pool_return() { let temp = tempfile::tempdir().expect("temp"); diff --git a/src/blaze/crates/blazed/src/checkpoint_store.rs b/src/blaze/crates/blazed/src/checkpoint_store.rs new file mode 100644 index 0000000000..fb0ec265c2 --- /dev/null +++ b/src/blaze/crates/blazed/src/checkpoint_store.rs @@ -0,0 +1,1180 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Filesystem-backed checkpoint catalog owned by the daemon. +//! +//! Publication and HEAD updates are separate durability boundaries. A +//! checkpoint can therefore be published but unreachable after an interrupted +//! operation; listing exposes that state for explicit inspection. + +use std::collections::{HashMap, HashSet}; +use std::fs::{self, File}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use blaze_core::checkpoint::{ + CHECKPOINT_FORMAT_VERSION, CheckpointArtifact, CheckpointInfo, CheckpointMetadata, + CheckpointValidationError, CommitCheckpoint, REQUIRED_ARTIFACTS, validate_artifact_name, + validate_checkpoint_id, validate_checkpoint_manifest, validate_commit_checkpoint, +}; +use chrono::Utc; +use sha2::{Digest, Sha256}; +use thiserror::Error; +use uuid::Uuid; + +const METADATA_FILE: &str = "metadata.json"; +const HEAD_FILE: &str = "HEAD"; +const STAGING_SUFFIX: &str = ".tmp"; +const TOMBSTONE_SUFFIX: &str = ".tombstone"; +const ABORT_TOMBSTONE_PREFIX: &str = ".abort."; + +/// Failure while reading or mutating the daemon checkpoint catalog. +#[derive(Debug, Error)] +pub enum CheckpointStoreError { + /// A checkpoint record failed pure model validation. + #[error(transparent)] + Validation(#[from] CheckpointValidationError), + + /// A catalog filesystem operation failed. + #[error("checkpoint catalog {operation} failed for {}: {source}", path.display())] + Io { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + + /// A metadata file could not be encoded or decoded. + #[error("checkpoint metadata at {} is invalid: {source}", path.display())] + Json { + path: PathBuf, + #[source] + source: serde_json::Error, + }, + + /// The catalog layout violates an invariant required for safe mutation. + #[error("checkpoint catalog invariant failed: {0}")] + Invariant(String), +} + +/// Convenient result type for checkpoint catalog operations. +pub type Result = std::result::Result; + +/// Temporary checkpoint directory populated before atomic publication. +#[derive(Debug)] +pub struct CheckpointStage { + id: String, + sandbox_id: Uuid, + path: PathBuf, + final_path: PathBuf, +} + +impl CheckpointStage { + /// Generated checkpoint identifier. + pub fn id(&self) -> &str { + &self.id + } + + /// Resolve one frozen artifact inside the stage. + pub fn artifact_path(&self, name: &str) -> Result { + validate_artifact_name(name)?; + Ok(self.path.join(name)) + } +} + +/// Filesystem-backed checkpoint catalog. +#[derive(Debug, Clone)] +pub struct CheckpointStore { + root: PathBuf, +} + +impl CheckpointStore { + /// Create a catalog rooted at `root` without performing I/O. + pub fn new(root: PathBuf) -> Self { + Self { root } + } + + /// Create and durably expose a unique staging directory. + pub fn begin(&self, sandbox_id: Uuid) -> Result { + let sandbox_dir = self.ensure_sandbox_dir(sandbox_id)?; + let id = format!("ckpt-{}", Uuid::new_v4()); + let path = sandbox_dir.join(format!(".{id}{STAGING_SUFFIX}")); + create_directory(&path, "create staging directory")?; + sync_directory(&sandbox_dir)?; + Ok(CheckpointStage { + final_path: sandbox_dir.join(&id), + id, + sandbox_id, + path, + }) + } + + /// Hash, sync, and atomically publish a populated stage without moving HEAD. + pub fn publish( + &self, + stage: &CheckpointStage, + input: CommitCheckpoint, + ) -> Result { + self.validate_stage(stage)?; + validate_commit_checkpoint(&stage.id, &input)?; + if let Some(parent) = &input.parent { + self.validated_chain_from(stage.sandbox_id, parent)?; + } + ensure_missing(&stage.final_path, "inspect publication target")?; + validate_exact_entries(&stage.path, &REQUIRED_ARTIFACTS)?; + + let mut artifacts = Vec::with_capacity(REQUIRED_ARTIFACTS.len()); + for name in REQUIRED_ARTIFACTS { + let path = require_contained_file(&stage.path, name)?; + sync_file(&path)?; + artifacts.push(hash_artifact(&path, name)?); + } + + let metadata = CheckpointMetadata { + format_version: CHECKPOINT_FORMAT_VERSION, + id: stage.id.clone(), + parent: input.parent, + sandbox_id: stage.sandbox_id, + policy_name: input.policy_name, + image_digest: input.image_digest, + backend: input.backend, + backend_version: input.backend_version, + created_at: Utc::now(), + snapshot_kind: input.snapshot_kind, + artifacts, + }; + validate_checkpoint_manifest(&metadata, stage.sandbox_id, &stage.id)?; + + let metadata_path = stage.path.join(METADATA_FILE); + write_json_new(&metadata_path, &metadata)?; + sync_directory(&stage.path)?; + rename_path( + &stage.path, + &stage.final_path, + "publish checkpoint directory", + )?; + let sandbox_dir = stage + .final_path + .parent() + .ok_or_else(|| invariant("published checkpoint has no sandbox parent"))?; + checkpoint_store_failpoint("checkpoint-store-publish-after-rename", &stage.final_path)?; + sync_directory(sandbox_dir)?; + Ok(metadata) + } + + /// Remove an unpublished stage owned by this process. + pub fn abort(&self, stage: CheckpointStage) -> Result<()> { + self.abort_staging(stage.sandbox_id, &stage.id) + } + + /// Remove one unpublished stage if it still exists. + /// + /// The stage is first renamed to a tombstone so a process interruption + /// cannot make it appear publishable again. Startup cleanup removes any + /// residual tombstone. + pub fn abort_staging(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result<()> { + validate_checkpoint_id(checkpoint_id)?; + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(()); + }; + let stage = sandbox_dir.join(format!(".{checkpoint_id}{STAGING_SUFFIX}")); + let Some(metadata) = optional_symlink_metadata(&stage, "inspect staging directory")? else { + return Ok(()); + }; + require_plain_directory_metadata(&stage, &metadata, "checkpoint staging directory")?; + require_direct_child(&sandbox_dir, &stage, "checkpoint staging directory")?; + + let tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, checkpoint_id); + rename_path(&stage, &tombstone, "tombstone aborted checkpoint stage")?; + sync_directory(&sandbox_dir)?; + remove_directory(&tombstone, "remove aborted checkpoint tombstone")?; + sync_directory(&sandbox_dir) + } + + /// Read and validate one committed checkpoint and all artifact hashes. + pub fn verify(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result { + let dir = self.committed_dir(sandbox_id, checkpoint_id)?; + validate_exact_entries( + &dir, + &[ + REQUIRED_ARTIFACTS[0], + REQUIRED_ARTIFACTS[1], + REQUIRED_ARTIFACTS[2], + METADATA_FILE, + ], + )?; + let metadata_path = require_contained_file(&dir, METADATA_FILE)?; + let bytes = read_file(&metadata_path, "read checkpoint metadata")?; + let metadata: CheckpointMetadata = + serde_json::from_slice(&bytes).map_err(|source| CheckpointStoreError::Json { + path: metadata_path, + source, + })?; + validate_checkpoint_manifest(&metadata, sandbox_id, checkpoint_id)?; + + for name in REQUIRED_ARTIFACTS { + let expected = metadata + .artifacts + .iter() + .find(|artifact| artifact.name == name) + .ok_or_else(|| { + invariant(format!( + "validated checkpoint {checkpoint_id} has no record for {name}" + )) + })?; + let path = require_contained_file(&dir, name)?; + let actual = hash_artifact(&path, name)?; + if &actual != expected { + return Err(invariant(format!( + "checkpoint {checkpoint_id} artifact {name} failed integrity validation" + ))); + } + } + Ok(metadata) + } + + /// List committed checkpoints and mark the lineage reachable from HEAD. + pub fn list(&self, sandbox_id: Uuid) -> Result> { + let Some(_) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + let catalog = self.load_catalog(sandbox_id)?; + let head = self.read_head(sandbox_id)?; + let on_head_chain = match head.as_deref() { + Some(head) => lineage_from(&catalog, head)?, + None => HashSet::new(), + }; + + let mut checkpoints = Vec::with_capacity(catalog.len()); + for metadata in catalog.into_values() { + let size_bytes = metadata + .artifacts + .iter() + .try_fold(0_u64, |total, artifact| { + total.checked_add(artifact.size_bytes) + }) + .ok_or_else(|| { + invariant(format!( + "checkpoint {} artifact sizes overflow u64", + metadata.id + )) + })?; + checkpoints.push(CheckpointInfo { + id: metadata.id.clone(), + parent: metadata.parent, + created_at: metadata.created_at, + size_bytes, + is_head: head.as_deref() == Some(metadata.id.as_str()), + on_head_chain: on_head_chain.contains(&metadata.id), + }); + } + checkpoints.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + Ok(checkpoints) + } + + /// Atomically move HEAD to an already committed, verified checkpoint. + pub fn set_head(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result<()> { + self.verify(sandbox_id, checkpoint_id)?; + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} disappeared")))?; + validate_existing_head_type(&sandbox_dir)?; + + let temporary = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + let outcome = (|| { + let mut file = open_new_file(&temporary, "create temporary HEAD")?; + write_all(&mut file, &temporary, checkpoint_id.as_bytes())?; + write_all(&mut file, &temporary, b"\n")?; + sync_open_file(&file, &temporary)?; + rename_path( + &temporary, + &sandbox_dir.join(HEAD_FILE), + "publish checkpoint HEAD", + )?; + checkpoint_store_failpoint( + "checkpoint-store-head-after-rename", + &sandbox_dir.join(HEAD_FILE), + )?; + sync_directory(&sandbox_dir) + })(); + if outcome.is_err() { + let _ = remove_file_if_exists(&temporary, "remove temporary HEAD"); + } + outcome + } + + /// Return the persisted HEAD, if present. + pub fn read_head(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(None); + }; + let path = sandbox_dir.join(HEAD_FILE); + let Some(metadata) = optional_symlink_metadata(&path, "inspect checkpoint HEAD")? else { + return Ok(None); + }; + require_plain_file_metadata(&path, &metadata, "checkpoint HEAD")?; + require_direct_child(&sandbox_dir, &path, "checkpoint HEAD")?; + let bytes = read_file(&path, "read checkpoint HEAD")?; + let raw = std::str::from_utf8(&bytes) + .map_err(|error| invariant(format!("checkpoint HEAD is not UTF-8: {error}")))?; + let checkpoint_id = raw + .strip_suffix('\n') + .filter(|value| !value.contains('\n') && !value.contains('\r')) + .ok_or_else(|| invariant("checkpoint HEAD is not one canonical line"))?; + validate_checkpoint_id(checkpoint_id)?; + self.committed_dir(sandbox_id, checkpoint_id)?; + Ok(Some(checkpoint_id.to_string())) + } + + /// Remove incomplete stages, temporary HEAD files, and cleanup tombstones. + /// + /// Committed checkpoint directories and the published HEAD are retained. + pub fn cleanup_transaction_artifacts(&self, sandbox_id: Uuid) -> Result> { + let scratch = self.transaction_artifacts(sandbox_id)?; + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + + let mut removed = Vec::with_capacity(scratch.len()); + for (path, kind) in scratch { + match kind { + ScratchKind::Directory => { + remove_directory(&path, "remove checkpoint scratch directory")? + } + ScratchKind::File => remove_file(&path, "remove checkpoint scratch file")?, + } + removed.push(path); + } + if !removed.is_empty() { + sync_directory(&sandbox_dir)?; + } + Ok(removed) + } + + fn transaction_artifacts(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(Vec::new()); + }; + let mut scratch = Vec::new(); + let entries = read_directory(&sandbox_dir, "scan checkpoint scratch")?; + for entry in entries { + let entry = entry + .map_err(|source| io_error("read checkpoint scratch", &sandbox_dir, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + let Some(kind) = classify_scratch_name(name)? else { + continue; + }; + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint scratch", &path, source))?; + match kind { + ScratchKind::Directory if file_type.is_dir() && !file_type.is_symlink() => { + scratch.push((path, ScratchKind::Directory)); + } + ScratchKind::File if file_type.is_file() && !file_type.is_symlink() => { + scratch.push((path, ScratchKind::File)); + } + _ => { + return Err(invariant(format!( + "checkpoint scratch {} has an unexpected file type", + path.display() + ))); + } + } + } + scratch.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(scratch) + } + + fn validate_stage(&self, stage: &CheckpointStage) -> Result<()> { + validate_checkpoint_id(&stage.id)?; + let sandbox_dir = self + .optional_sandbox_dir(stage.sandbox_id)? + .ok_or_else(|| invariant("checkpoint staging sandbox does not exist"))?; + let expected_path = sandbox_dir.join(format!(".{}{STAGING_SUFFIX}", stage.id)); + let expected_final_path = sandbox_dir.join(&stage.id); + if stage.path != expected_path || stage.final_path != expected_final_path { + return Err(invariant( + "checkpoint stage paths do not match its frozen identity", + )); + } + require_plain_directory(&stage.path, "checkpoint staging directory")?; + require_direct_child(&sandbox_dir, &stage.path, "checkpoint staging directory") + } + + fn validated_chain_from(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result> { + validate_checkpoint_id(checkpoint_id)?; + let mut current = checkpoint_id.to_string(); + let mut lineage = Vec::new(); + let mut seen = HashSet::new(); + loop { + if !seen.insert(current.clone()) { + return Err(invariant(format!( + "checkpoint parent cycle reaches {current}" + ))); + } + let metadata = self.verify(sandbox_id, ¤t)?; + lineage.push(current); + let Some(parent) = metadata.parent else { + break; + }; + current = parent; + } + Ok(lineage) + } + + fn load_catalog(&self, sandbox_id: Uuid) -> Result> { + let Some(sandbox_dir) = self.optional_sandbox_dir(sandbox_id)? else { + return Ok(HashMap::new()); + }; + let mut catalog = HashMap::new(); + let entries = read_directory(&sandbox_dir, "scan checkpoint catalog")?; + for entry in entries { + let entry = entry + .map_err(|source| io_error("read checkpoint catalog", &sandbox_dir, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + continue; + }; + if !name.starts_with("ckpt-") { + continue; + } + validate_checkpoint_id(name)?; + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint entry", entry.path(), source))?; + if !file_type.is_dir() || file_type.is_symlink() { + return Err(invariant(format!( + "checkpoint entry {} is not a plain directory", + entry.path().display() + ))); + } + let metadata = self.verify(sandbox_id, name)?; + catalog.insert(name.to_string(), metadata); + } + Ok(catalog) + } + + fn ensure_sandbox_dir(&self, sandbox_id: Uuid) -> Result { + let root_was_missing = + optional_symlink_metadata(&self.root, "inspect checkpoint root")?.is_none(); + create_directories(&self.root, "create checkpoint root")?; + require_plain_directory(&self.root, "checkpoint root")?; + if root_was_missing + && let Some(parent) = self + .root + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + sync_directory(parent)?; + } + let sandbox_dir = self.sandbox_dir(sandbox_id); + match fs::create_dir(&sandbox_dir) { + Ok(()) => { + sync_directory(&self.root)?; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(io_error( + "create checkpoint sandbox directory", + &sandbox_dir, + source, + )); + } + } + require_plain_directory(&sandbox_dir, "checkpoint sandbox directory")?; + require_direct_child(&self.root, &sandbox_dir, "checkpoint sandbox directory")?; + Ok(sandbox_dir) + } + + fn optional_sandbox_dir(&self, sandbox_id: Uuid) -> Result> { + let Some(root_metadata) = optional_symlink_metadata(&self.root, "inspect checkpoint root")? + else { + return Ok(None); + }; + require_plain_directory_metadata(&self.root, &root_metadata, "checkpoint root")?; + let sandbox_dir = self.sandbox_dir(sandbox_id); + let Some(metadata) = + optional_symlink_metadata(&sandbox_dir, "inspect checkpoint sandbox directory")? + else { + return Ok(None); + }; + require_plain_directory_metadata(&sandbox_dir, &metadata, "checkpoint sandbox directory")?; + require_direct_child(&self.root, &sandbox_dir, "checkpoint sandbox directory")?; + Ok(Some(sandbox_dir)) + } + + fn committed_dir(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result { + validate_checkpoint_id(checkpoint_id)?; + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} does not exist")))?; + let path = sandbox_dir.join(checkpoint_id); + require_plain_directory(&path, "committed checkpoint directory")?; + require_direct_child(&sandbox_dir, &path, "committed checkpoint directory")?; + Ok(path) + } + + fn sandbox_dir(&self, sandbox_id: Uuid) -> PathBuf { + self.root.join(sandbox_id.to_string()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScratchKind { + Directory, + File, +} + +fn lineage_from( + catalog: &HashMap, + checkpoint_id: &str, +) -> Result> { + validate_checkpoint_id(checkpoint_id)?; + let mut current = checkpoint_id; + let mut lineage = HashSet::new(); + loop { + if !lineage.insert(current.to_string()) { + return Err(invariant(format!( + "checkpoint parent cycle reaches {current}" + ))); + } + let metadata = catalog.get(current).ok_or_else(|| { + invariant(format!( + "checkpoint lineage references missing checkpoint {current}" + )) + })?; + let Some(parent) = metadata.parent.as_deref() else { + break; + }; + current = parent; + } + Ok(lineage) +} + +fn validate_exact_entries(directory: &Path, expected: &[&str]) -> Result<()> { + let expected = expected.iter().copied().collect::>(); + let mut observed = HashSet::new(); + let entries = read_directory(directory, "scan checkpoint directory")?; + for entry in entries { + let entry = + entry.map_err(|source| io_error("read checkpoint directory", directory, source))?; + let name = entry.file_name(); + let Some(name) = name.to_str() else { + return Err(invariant(format!( + "checkpoint directory {} contains a non-UTF-8 entry", + directory.display() + ))); + }; + let file_type = entry + .file_type() + .map_err(|source| io_error("inspect checkpoint entry", entry.path(), source))?; + if !expected.contains(name) || !file_type.is_file() || file_type.is_symlink() { + return Err(invariant(format!( + "checkpoint directory {} contains unexpected entry {name:?}", + directory.display() + ))); + } + observed.insert(name.to_string()); + } + if observed.len() != expected.len() || expected.iter().any(|name| !observed.contains(*name)) { + return Err(invariant(format!( + "checkpoint directory {} does not contain the exact required file set", + directory.display() + ))); + } + Ok(()) +} + +fn hash_artifact(path: &Path, name: &str) -> Result { + let mut file = open_file(path, "open checkpoint artifact")?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 128 * 1024]; + loop { + let read = file + .read(&mut buffer) + .map_err(|source| io_error("read checkpoint artifact", path, source))?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let size_bytes = file + .metadata() + .map_err(|source| io_error("inspect checkpoint artifact", path, source))? + .len(); + Ok(CheckpointArtifact { + name: name.to_string(), + size_bytes, + sha256: format!("{:x}", hasher.finalize()), + }) +} + +fn classify_scratch_name(name: &str) -> Result> { + if let Some(checkpoint_id) = name + .strip_prefix('.') + .and_then(|name| name.strip_suffix(STAGING_SUFFIX)) + .filter(|name| name.starts_with("ckpt-")) + { + validate_checkpoint_id(checkpoint_id)?; + return Ok(Some(ScratchKind::Directory)); + } + if let Some(nonce) = name + .strip_prefix(".HEAD.") + .and_then(|name| name.strip_suffix(STAGING_SUFFIX)) + { + parse_uuid_component(nonce, "temporary HEAD")?; + return Ok(Some(ScratchKind::File)); + } + if let Some(body) = name + .strip_prefix(ABORT_TOMBSTONE_PREFIX) + .and_then(|name| name.strip_suffix(TOMBSTONE_SUFFIX)) + { + let (checkpoint_id, nonce) = body + .rsplit_once('.') + .ok_or_else(|| invariant(format!("invalid checkpoint tombstone {name:?}")))?; + validate_checkpoint_id(checkpoint_id)?; + parse_uuid_component(nonce, "checkpoint tombstone")?; + return Ok(Some(ScratchKind::Directory)); + } + Ok(None) +} + +fn parse_uuid_component(value: &str, label: &str) -> Result { + let uuid = Uuid::parse_str(value) + .map_err(|error| invariant(format!("invalid {label} identifier {value:?}: {error}")))?; + if value != uuid.to_string() { + return Err(invariant(format!( + "{label} identifier {value:?} is not canonical" + ))); + } + Ok(uuid) +} + +fn tombstone_path(directory: &Path, prefix: &str, checkpoint_id: &str) -> PathBuf { + directory.join(format!( + "{prefix}{checkpoint_id}.{}{TOMBSTONE_SUFFIX}", + Uuid::new_v4() + )) +} + +fn validate_existing_head_type(sandbox_dir: &Path) -> Result<()> { + let path = sandbox_dir.join(HEAD_FILE); + let Some(metadata) = optional_symlink_metadata(&path, "inspect existing checkpoint HEAD")? + else { + return Ok(()); + }; + require_plain_file_metadata(&path, &metadata, "existing checkpoint HEAD") +} + +fn require_plain_directory(path: &Path, label: &str) -> Result<()> { + let metadata = symlink_metadata(path, "inspect directory")?; + require_plain_directory_metadata(path, &metadata, label) +} + +fn require_plain_directory_metadata( + path: &Path, + metadata: &fs::Metadata, + label: &str, +) -> Result<()> { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err(invariant(format!( + "{label} {} is not a plain directory", + path.display() + ))); + } + Ok(()) +} + +fn require_plain_file_metadata(path: &Path, metadata: &fs::Metadata, label: &str) -> Result<()> { + if !metadata.is_file() || metadata.file_type().is_symlink() { + return Err(invariant(format!( + "{label} {} is not a plain file", + path.display() + ))); + } + Ok(()) +} + +fn require_contained_file(directory: &Path, name: &str) -> Result { + if name != METADATA_FILE { + validate_artifact_name(name)?; + } + let path = directory.join(name); + let metadata = symlink_metadata(&path, "inspect checkpoint file")?; + require_plain_file_metadata(&path, &metadata, "checkpoint file")?; + require_direct_child(directory, &path, "checkpoint file")?; + Ok(path) +} + +fn require_direct_child(parent: &Path, child: &Path, label: &str) -> Result<()> { + let canonical_parent = canonicalize(parent, "canonicalize checkpoint parent")?; + let canonical_child = canonicalize(child, "canonicalize checkpoint child")?; + if canonical_child.parent() != Some(canonical_parent.as_path()) { + return Err(invariant(format!( + "{label} {} is not directly contained by {}", + child.display(), + parent.display() + ))); + } + Ok(()) +} + +fn ensure_missing(path: &Path, operation: &'static str) -> Result<()> { + match fs::symlink_metadata(path) { + Ok(_) => Err(invariant(format!( + "checkpoint publication target {} already exists", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn optional_symlink_metadata(path: &Path, operation: &'static str) -> Result> { + match fs::symlink_metadata(path) { + Ok(metadata) => Ok(Some(metadata)), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn symlink_metadata(path: &Path, operation: &'static str) -> Result { + fs::symlink_metadata(path).map_err(|source| io_error(operation, path, source)) +} + +fn create_directories(path: &Path, operation: &'static str) -> Result<()> { + fs::create_dir_all(path).map_err(|source| io_error(operation, path, source)) +} + +fn create_directory(path: &Path, operation: &'static str) -> Result<()> { + fs::create_dir(path).map_err(|source| io_error(operation, path, source)) +} + +fn open_file(path: &Path, operation: &'static str) -> Result { + File::open(path).map_err(|source| io_error(operation, path, source)) +} + +fn open_new_file(path: &Path, operation: &'static str) -> Result { + fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(path) + .map_err(|source| io_error(operation, path, source)) +} + +fn write_all(file: &mut File, path: &Path, bytes: &[u8]) -> Result<()> { + file.write_all(bytes) + .map_err(|source| io_error("write file", path, source)) +} + +fn sync_open_file(file: &File, path: &Path) -> Result<()> { + file.sync_all() + .map_err(|source| io_error("sync file", path, source)) +} + +fn write_json_new(path: &Path, value: &T) -> Result<()> { + let bytes = serde_json::to_vec_pretty(value).map_err(|source| CheckpointStoreError::Json { + path: path.to_path_buf(), + source, + })?; + let mut file = open_new_file(path, "create checkpoint metadata")?; + write_all(&mut file, path, &bytes)?; + write_all(&mut file, path, b"\n")?; + sync_open_file(&file, path) +} + +fn read_file(path: &Path, operation: &'static str) -> Result> { + fs::read(path).map_err(|source| io_error(operation, path, source)) +} + +fn read_directory(path: &Path, operation: &'static str) -> Result { + fs::read_dir(path).map_err(|source| io_error(operation, path, source)) +} + +fn canonicalize(path: &Path, operation: &'static str) -> Result { + fs::canonicalize(path).map_err(|source| io_error(operation, path, source)) +} + +fn rename_path(source: &Path, target: &Path, operation: &'static str) -> Result<()> { + fs::rename(source, target).map_err(|error| { + io_error( + operation, + PathBuf::from(format!("{} -> {}", source_path(source), target.display())), + error, + ) + }) +} + +fn source_path(path: &Path) -> String { + path.display().to_string() +} + +fn sync_file(path: &Path) -> Result<()> { + let file = open_file(path, "open file for sync")?; + sync_open_file(&file, path) +} + +fn sync_directory(path: &Path) -> Result<()> { + let directory = open_file(path, "open directory for sync")?; + directory + .sync_all() + .map_err(|source| io_error("sync directory", path, source)) +} + +fn remove_directory(path: &Path, operation: &'static str) -> Result<()> { + fs::remove_dir_all(path).map_err(|source| io_error(operation, path, source)) +} + +fn remove_file(path: &Path, operation: &'static str) -> Result<()> { + fs::remove_file(path).map_err(|source| io_error(operation, path, source)) +} + +fn remove_file_if_exists(path: &Path, operation: &'static str) -> Result<()> { + match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(source) => Err(io_error(operation, path, source)), + } +} + +fn io_error( + operation: &'static str, + path: impl AsRef, + source: std::io::Error, +) -> CheckpointStoreError { + CheckpointStoreError::Io { + operation, + path: path.as_ref().to_path_buf(), + source, + } +} + +fn invariant(message: impl Into) -> CheckpointStoreError { + CheckpointStoreError::Invariant(message.into()) +} + +fn checkpoint_store_failpoint(name: &'static str, path: &Path) -> Result<()> { + crate::failpoint::storage(name).map_err(|error| { + io_error( + "run checkpoint store failpoint", + path, + std::io::Error::other(error.to_string()), + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use blaze_core::backend::{BackendKind, SnapshotKind}; + + fn commit_input(parent: Option) -> CommitCheckpoint { + CommitCheckpoint { + parent, + policy_name: "default".to_string(), + image_digest: "sha256:test".to_string(), + backend: BackendKind::Mock, + backend_version: Some("mock-v1".to_string()), + snapshot_kind: SnapshotKind::Full, + } + } + + fn populate(stage: &CheckpointStage, suffix: &str) { + for name in REQUIRED_ARTIFACTS { + fs::write(stage.path.join(name), format!("{name}-{suffix}")).expect("write artifact"); + } + } + + fn publish( + store: &CheckpointStore, + sandbox_id: Uuid, + parent: Option, + move_head: bool, + ) -> String { + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let id = stage.id().to_string(); + populate(&stage, &id); + store + .publish(&stage, commit_input(parent)) + .expect("publish checkpoint"); + if move_head { + store.set_head(sandbox_id, &id).expect("move HEAD"); + } + id + } + + #[test] + fn publish_verify_and_list_preserve_the_head_boundary() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + store + .verify(sandbox_id, &unreachable) + .expect("published checkpoint"); + let listed = store.list(sandbox_id).expect("list checkpoints"); + assert_eq!(listed.len(), 2); + assert_eq!(listed.iter().filter(|info| info.is_head).count(), 1); + assert!( + listed + .iter() + .any(|info| info.id == unreachable && !info.on_head_chain) + ); + } + + #[test] + fn verify_rejects_corrupted_artifact_content() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, true); + fs::write( + store + .root + .join(sandbox_id.to_string()) + .join(&checkpoint_id) + .join("memory.snap"), + b"corrupt", + ) + .expect("corrupt artifact"); + + assert!(store.verify(sandbox_id, &checkpoint_id).is_err()); + } + + #[test] + fn publish_rejects_an_unexpected_stage_entry() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + populate(&stage, "candidate"); + fs::write(stage.path.join("unexpected"), b"unexpected").expect("write extra entry"); + + let error = store + .publish(&stage, commit_input(None)) + .expect_err("unexpected entry must fail"); + assert!(error.to_string().contains("unexpected entry")); + store + .abort_staging(sandbox_id, stage.id()) + .expect("abort stage"); + } + + #[test] + fn cleanup_removes_transaction_scratch_but_retains_committed_history() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let committed = publish(&store, sandbox_id, None, true); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let sandbox_dir = store.root.join(sandbox_id.to_string()); + let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); + let tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, &committed); + fs::create_dir(&tombstone).expect("create abort tombstone"); + + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup scratch"); + + assert_eq!(removed.len(), 3); + assert!(!stage.path.exists()); + assert!(!temporary_head.exists()); + assert!(!tombstone.exists()); + assert!(store.verify(sandbox_id, &committed).is_ok()); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(committed)); + } + + #[test] + fn cleanup_recognizes_every_capture_transaction_artifact() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let sandbox_dir = store.root.join(sandbox_id.to_string()); + let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); + fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); + let abort_tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, stage.id()); + fs::create_dir(&abort_tombstone).expect("create abort tombstone"); + + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup transaction artifacts"); + assert_eq!(removed.len(), 3); + assert!(!stage.path.exists()); + assert!(!temporary_head.exists()); + assert!(!abort_tombstone.exists()); + } + + #[test] + fn cleanup_reports_unsafe_layout_without_deleting_it() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + fs::remove_dir(&stage.path).expect("remove staging directory"); + fs::write(&stage.path, b"not a directory").expect("replace stage with file"); + + let cleanup_error = store + .cleanup_transaction_artifacts(sandbox_id) + .expect_err("cleanup must not delete an entry with the wrong type"); + assert!(cleanup_error.to_string().contains("unexpected file type")); + assert!(stage.path.is_file()); + } + + #[test] + fn abort_staging_is_idempotent() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let checkpoint_id = stage.id().to_string(); + + store + .abort_staging(sandbox_id, &checkpoint_id) + .expect("abort stage"); + store + .abort_staging(sandbox_id, &checkpoint_id) + .expect("repeat abort"); + assert!(!stage.path.exists()); + } + + #[test] + fn missing_catalog_has_no_head_or_entries() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("missing")); + let sandbox_id = Uuid::new_v4(); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); + assert!(store.list(sandbox_id).expect("list").is_empty()); + assert!( + store + .cleanup_transaction_artifacts(sandbox_id) + .expect("cleanup") + .is_empty() + ); + } + + #[cfg(unix)] + #[test] + fn verify_rejects_artifact_symlinks() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, true); + let artifact = store + .root + .join(sandbox_id.to_string()) + .join(&checkpoint_id) + .join("rootfs.snap"); + fs::remove_file(&artifact).expect("remove artifact"); + let outside = temp.path().join("outside"); + fs::write(&outside, b"outside").expect("write outside file"); + symlink(&outside, &artifact).expect("link artifact"); + + assert!(store.verify(sandbox_id, &checkpoint_id).is_err()); + } + + #[cfg(unix)] + #[test] + fn begin_rejects_a_symlinked_catalog_root() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let actual = temp.path().join("actual"); + fs::create_dir(&actual).expect("create actual root"); + let linked = temp.path().join("linked"); + symlink(&actual, &linked).expect("link root"); + let store = CheckpointStore::new(linked); + + assert!(store.begin(Uuid::new_v4()).is_err()); + } + + #[cfg(not(feature = "test-failpoints"))] + #[test] + fn production_checkpoint_store_boundary_hooks_are_inert() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn publish_boundary_error_leaves_a_committed_unreachable_checkpoint() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let stage = store.begin(sandbox_id).expect("begin checkpoint"); + let checkpoint_id = stage.id().to_string(); + let final_path = stage.final_path.clone(); + populate(&stage, "publish-boundary"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-publish-after-rename"]); + + let error = hook + .run(async { store.publish(&stage, commit_input(None)) }) + .await + .expect_err("publish boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-store-publish-after-rename") + ); + assert!(!stage.path.exists()); + assert!(final_path.is_dir()); + store + .verify(sandbox_id, &checkpoint_id) + .expect("renamed checkpoint remains committed"); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); + assert!( + store + .cleanup_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + .is_empty() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn head_boundary_error_leaves_the_new_head_visible() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let checkpoint_id = publish(&store, sandbox_id, None, false); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-head-after-rename"]); + + let error = hook + .run(async { store.set_head(sandbox_id, &checkpoint_id) }) + .await + .expect_err("HEAD boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-store-head-after-rename") + ); + assert_eq!( + store.read_head(sandbox_id).expect("HEAD"), + Some(checkpoint_id) + ); + assert!( + store + .cleanup_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + .is_empty() + ); + } +} diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index fb78fdb67b..84fafce120 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -3,9 +3,11 @@ //! rootfs and memory files on a local filesystem. Base images and mutable //! instance slots use separate roots; runtime pooling is owned by the daemon. -use std::path::{Path, PathBuf}; +use std::ffi::OsString; +use std::path::{Component, Path, PathBuf}; use async_trait::async_trait; +use uuid::Uuid; use blaze_core::error::{BlazeError, Result}; use blaze_core::storage::{ @@ -81,6 +83,50 @@ impl RequiredPathType { } } +/// Removes incomplete capture files if an error or cancellation interrupts +/// publication before the target directory is durably synchronized. +struct UnpublishedCheckpoint { + temporary: Option, + target: Option, +} + +impl UnpublishedCheckpoint { + fn new() -> Self { + Self { + temporary: None, + target: None, + } + } + + fn mark_temporary(&mut self, temporary: PathBuf) { + self.temporary = Some(temporary); + } + + fn mark_target(&mut self, target: PathBuf) { + self.target = Some(target); + } + + fn clear_temporary(&mut self) { + self.temporary = None; + } + + fn commit(&mut self) { + self.temporary = None; + self.target = None; + } +} + +impl Drop for UnpublishedCheckpoint { + fn drop(&mut self) { + if let Some(target) = self.target.take() { + let _ = std::fs::remove_file(target); + } + if let Some(temporary) = self.temporary.take() { + let _ = std::fs::remove_file(temporary); + } + } +} + async fn require_slot_path( instance_id: &str, path: &Path, @@ -283,6 +329,29 @@ impl StorageProvider for FileStorageProvider { Ok(()) } + fn supports_checkpoint_capture(&self) -> bool { + true + } + + async fn capture_checkpoint(&self, slot: &StorageSlot, target: &Path) -> Result<()> { + let source = self.checkpoint_source(slot).await?; + let (target_parent, target) = checkpoint_target(target).await?; + ensure_checkpoint_target_absent(&target).await?; + + let temporary = checkpoint_temporary_path(&target_parent, &target); + let mut cleanup = UnpublishedCheckpoint::new(); + let result = + capture_rootfs(&source, &temporary, &target_parent, &target, &mut cleanup).await; + result.map_err(|error| BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': copy {} to {}: {error}", + slot.id, + source.display(), + target.display() + ), + }) + } + fn pool_status(&self) -> PoolStatus { PoolStatus::default() } @@ -292,6 +361,41 @@ impl StorageProvider for FileStorageProvider { } } +impl FileStorageProvider { + async fn checkpoint_source(&self, slot: &StorageSlot) -> Result { + let canonical = self.slot_for_id(&slot.id)?; + let instances_dir = + canonical_plain_path(&self.instances_dir, RequiredPathType::Directory).await?; + let instance_dir = + canonical_plain_path(&canonical.instance_dir, RequiredPathType::Directory).await?; + if instance_dir.parent() != Some(instances_dir.as_path()) { + return Err(BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': slot {} is outside instances directory {}", + slot.id, + instance_dir.display(), + instances_dir.display() + ), + }); + } + + let source = canonical_plain_path(&canonical.rootfs_path, RequiredPathType::File).await?; + if source.parent() != Some(instance_dir.as_path()) + || source.file_name() != canonical.rootfs_path.file_name() + { + return Err(BlazeError::StorageError { + msg: format!( + "capture checkpoint for '{}': rootfs {} is outside slot {}", + slot.id, + source.display(), + instance_dir.display() + ), + }); + } + Ok(source) + } +} + async fn create_or_copy( source: &std::path::Path, target: &std::path::Path, @@ -308,6 +412,111 @@ async fn create_or_copy( Ok(()) } +async fn canonical_plain_path(path: &Path, required_type: RequiredPathType) -> Result { + let metadata = + tokio::fs::symlink_metadata(path) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("inspect checkpoint path {}: {error}", path.display()), + })?; + if !required_type.matches(&metadata) || metadata.file_type().is_symlink() { + return Err(BlazeError::StorageError { + msg: format!( + "checkpoint path {} is not a plain {}", + path.display(), + required_type.description() + ), + }); + } + tokio::fs::canonicalize(path) + .await + .map_err(|error| BlazeError::StorageError { + msg: format!("canonicalize checkpoint path {}: {error}", path.display()), + }) +} + +async fn checkpoint_target(target: &Path) -> Result<(PathBuf, PathBuf)> { + if !matches!(target.components().next_back(), Some(Component::Normal(_))) { + return Err(BlazeError::StorageError { + msg: format!( + "checkpoint target {} must end in a file name", + target.display() + ), + }); + } + let parent = target.parent().ok_or_else(|| BlazeError::StorageError { + msg: format!( + "checkpoint target {} has no parent directory", + target.display() + ), + })?; + let parent = canonical_plain_path(parent, RequiredPathType::Directory).await?; + let file_name = target.file_name().ok_or_else(|| BlazeError::StorageError { + msg: format!("checkpoint target {} has no file name", target.display()), + })?; + let target = parent.join(file_name); + Ok((parent, target)) +} + +async fn ensure_checkpoint_target_absent(target: &Path) -> Result<()> { + match tokio::fs::symlink_metadata(target).await { + Ok(_) => Err(BlazeError::StorageError { + msg: format!("checkpoint target {} already exists", target.display()), + }), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(BlazeError::StorageError { + msg: format!("inspect checkpoint target {}: {error}", target.display()), + }), + } +} + +fn checkpoint_temporary_path(parent: &Path, target: &Path) -> PathBuf { + let mut name = OsString::from("."); + name.push(target.file_name().expect("validated checkpoint target")); + name.push(format!(".capture-{}.tmp", Uuid::new_v4())); + parent.join(name) +} + +async fn capture_rootfs( + source: &Path, + temporary: &Path, + parent: &Path, + target: &Path, + cleanup: &mut UnpublishedCheckpoint, +) -> std::io::Result<()> { + let mut source_options = tokio::fs::OpenOptions::new(); + source_options.read(true); + #[cfg(unix)] + source_options.custom_flags(libc::O_NOFOLLOW); + let mut source_file = source_options.open(source).await?; + if !source_file.metadata().await?.is_file() { + return Err(std::io::Error::other(format!( + "checkpoint source {} is not a regular file", + source.display() + ))); + } + let mut temporary_file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(temporary) + .await?; + cleanup.mark_temporary(temporary.to_path_buf()); + tokio::io::copy(&mut source_file, &mut temporary_file).await?; + temporary_file.sync_all().await?; + drop(temporary_file); + drop(source_file); + + tokio::fs::hard_link(temporary, target).await?; + cleanup.mark_target(target.to_path_buf()); + crate::failpoint::storage("storage-capture-after-link") + .map_err(|error| std::io::Error::other(error.to_string()))?; + tokio::fs::remove_file(temporary).await?; + cleanup.clear_temporary(); + tokio::fs::File::open(parent).await?.sync_all().await?; + cleanup.commit(); + Ok(()) +} + fn validate_instance_id(instance_id: &str) -> Result<()> { if instance_id.is_empty() || instance_id.contains('/') @@ -327,6 +536,26 @@ fn validate_instance_id(instance_id: &str) -> Result<()> { mod tests { use super::*; + async fn checkpoint_fixture( + instance_id: &str, + ) -> (tempfile::TempDir, FileStorageProvider, StorageSlot, PathBuf) { + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let checkpoints = temp.path().join("checkpoints"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&checkpoints).await.unwrap(); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + (temp, provider, slot, checkpoints) + } + #[tokio::test] async fn probe_existing_dir_returns_true() { let tmp = tempfile::TempDir::new().unwrap(); @@ -727,4 +956,191 @@ mod tests { .expect_err("missing artifact must fail the sweep item"); assert!(error.to_string().contains("mem.diff"), "{error}"); } + + #[tokio::test] + async fn checkpoint_capture_is_explicit_and_independent() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-independent").await; + tokio::fs::write(&slot.rootfs_path, b"captured-rootfs") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + + assert!(provider.supports_checkpoint_capture()); + provider.capture_checkpoint(&slot, &target).await.unwrap(); + tokio::fs::write(&slot.rootfs_path, b"changed-live-rootfs") + .await + .unwrap(); + + assert_eq!(tokio::fs::read(&target).await.unwrap(), b"captured-rootfs"); + } + + #[tokio::test] + async fn checkpoint_capture_does_not_replace_the_live_rootfs() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-read-only").await; + tokio::fs::write(&slot.rootfs_path, b"live-rootfs") + .await + .unwrap(); + + provider + .capture_checkpoint(&slot, &checkpoints.join("rootfs.snap")) + .await + .unwrap(); + + assert_eq!( + tokio::fs::read(&slot.rootfs_path).await.unwrap(), + b"live-rootfs" + ); + } + + #[tokio::test] + async fn checkpoint_capture_ignores_forged_slot_paths() { + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-canonical").await; + tokio::fs::write(&slot.rootfs_path, b"canonical-rootfs") + .await + .unwrap(); + let forged_source = temp.path().join("forged-rootfs"); + tokio::fs::write(&forged_source, b"forged-rootfs") + .await + .unwrap(); + let mut forged = slot.clone(); + forged.rootfs_path = forged_source; + forged.mem_path = temp.path().join("forged-memory"); + forged.mem_diff_path = temp.path().join("forged-memory-diff"); + forged.rootfs_diff_path = temp.path().join("forged-rootfs-diff"); + forged.instance_dir = temp.path().to_path_buf(); + let target = checkpoints.join("rootfs.snap"); + + provider.capture_checkpoint(&forged, &target).await.unwrap(); + + assert_eq!(tokio::fs::read(&target).await.unwrap(), b"canonical-rootfs"); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_rootfs() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-linked-source").await; + tokio::fs::remove_file(&slot.rootfs_path).await.unwrap(); + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external").await.unwrap(); + symlink(&external, &slot.rootfs_path).unwrap(); + let target = checkpoints.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked rootfs must not be captured"); + + assert!(!target.exists()); + assert_eq!(tokio::fs::read(external).await.unwrap(), b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_slot_directory() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, checkpoints) = checkpoint_fixture("capture-linked-slot").await; + tokio::fs::remove_dir_all(&slot.instance_dir).await.unwrap(); + let external = temp.path().join("external-slot"); + tokio::fs::create_dir(&external).await.unwrap(); + tokio::fs::write(external.join("rootfs.ext4"), b"external") + .await + .unwrap(); + symlink(&external, &slot.instance_dir).unwrap(); + let target = checkpoints.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked slot directory must be rejected"); + + assert!(!target.exists()); + assert_eq!( + tokio::fs::read(external.join("rootfs.ext4")).await.unwrap(), + b"external" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn checkpoint_capture_rejects_a_linked_target_parent() { + use std::os::unix::fs::symlink; + + let temp = tempfile::TempDir::new().unwrap(); + let instances = temp.path().join("instances"); + let external = temp.path().join("external-checkpoints"); + tokio::fs::create_dir(&instances).await.unwrap(); + tokio::fs::create_dir(&external).await.unwrap(); + let linked_parent = temp.path().join("linked-checkpoints"); + symlink(&external, &linked_parent).unwrap(); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: "capture-linked-parent".into(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .unwrap(); + let target = linked_parent.join("rootfs.snap"); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("linked target parent must be rejected"); + + assert!(!external.join("rootfs.snap").exists()); + } + + #[tokio::test] + async fn checkpoint_capture_preserves_an_existing_target() { + let (_temp, provider, slot, checkpoints) = + checkpoint_fixture("capture-existing-target").await; + tokio::fs::write(&slot.rootfs_path, b"new-checkpoint") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + tokio::fs::write(&target, b"existing-checkpoint") + .await + .unwrap(); + + provider + .capture_checkpoint(&slot, &target) + .await + .expect_err("capture must never replace an existing target"); + + assert_eq!( + tokio::fs::read(&target).await.unwrap(), + b"existing-checkpoint" + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_capture_cleans_temporary_data_after_failure() { + let (_temp, provider, slot, checkpoints) = checkpoint_fixture("capture-cleanup").await; + tokio::fs::write(&slot.rootfs_path, b"complete-temporary-copy") + .await + .unwrap(); + let target = checkpoints.join("rootfs.snap"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-capture-after-link"]); + + hook.run(provider.capture_checkpoint(&slot, &target)) + .await + .expect_err("armed capture must roll back its unpublished target"); + + assert!(!target.exists()); + assert!( + tokio::fs::read_dir(&checkpoints) + .await + .unwrap() + .next_entry() + .await + .unwrap() + .is_none(), + "capture failure must remove its temporary file" + ); + } } diff --git a/src/blaze/crates/blazed/src/main.rs b/src/blaze/crates/blazed/src/main.rs index f333027bba..b678f65ab5 100644 --- a/src/blaze/crates/blazed/src/main.rs +++ b/src/blaze/crates/blazed/src/main.rs @@ -5,6 +5,7 @@ //! exposed via the HTTP API; this binary only handles daemon lifecycle. mod api; +mod checkpoint_store; mod cli; mod daemon; mod daemon_socket; diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs index 2952f6f3ab..93504747dd 100644 --- a/src/blaze/crates/blazed/src/sandbox.rs +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -1,6 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Managed sandbox lifecycle and runtime ownership. +mod checkpoint; mod manager; pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; diff --git a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs new file mode 100644 index 0000000000..cf8476a085 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs @@ -0,0 +1,442 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable checkpoint capture and listing. + +use blaze_core::backend::{SnapshotKind, SnapshotRequest}; +use blaze_core::checkpoint::{CheckpointInfo, CheckpointMetadata, CommitCheckpoint}; +use blaze_core::lifecycle::{OperationPhase, SandboxInstance, SandboxState}; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +impl SandboxManager { + /// Capture a self-contained checkpoint and resume the existing backend. + pub async fn checkpoint(&self, id: Uuid) -> Result { + let operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != SandboxState::Running { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected running", + instance.state + ))); + } + + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if !backend.supports_checkpoint_capture() || !self.storage.supports_checkpoint_capture() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} and configured storage do not support checkpoint capture", + backend.backend() + ))); + } + if backend.instance_id() != id || backend.backend() != instance.backend { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + let backend_version = backend.version().map(str::to_string); + if backend_version + .as_deref() + .is_some_and(|version| version.trim().is_empty()) + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not report a usable checkpoint version", + backend.backend() + ))); + } + self.require_live_backend(id, &backend).await?; + let storage = self.storage.reconstruct(&id.to_string()).await?; + + let stage = self.checkpoints.begin(id).map_err(checkpoint_store_error)?; + let checkpoint_id = stage.id().to_string(); + let snapshot_path = match stage.artifact_path("vmstate.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + let memory_path = match stage.artifact_path("memory.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + let rootfs_path = match stage.artifact_path("rootfs.snap") { + Ok(path) => path, + Err(error) => { + let _ = self.checkpoints.abort(stage); + return Err(checkpoint_store_error(error)); + } + }; + if let Err(error) = crate::failpoint::state("checkpoint-begin-state") { + let _ = self.checkpoints.abort(stage); + return Err(error); + } + if let Err(error) = instance.begin_checkpoint_operation(checkpoint_id.clone()) { + let _ = self.checkpoints.abort(stage); + return Err(error.into()); + } + if let Err(error) = self.persist_and_retain(instance.clone()) { + let _ = self.checkpoints.abort(stage); + return Err(error); + } + crate::failpoint::pause("checkpoint-after-begin").await; + + let paused = match crate::failpoint::backend("checkpoint-pause") { + Ok(()) => backend.pause().await, + Err(error) => Err(error), + }; + if let Err(error) = paused { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + if let Err(error) = instance + .transition(SandboxState::Paused) + .and_then(|_| instance.advance_checkpoint_phase(OperationPhase::CheckpointPaused)) + { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-paused-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error) + .await; + } + crate::failpoint::pause("checkpoint-after-pause").await; + + let snapshot = SnapshotRequest { + snapshot_path, + mem_path: memory_path, + kind: SnapshotKind::Full, + }; + let snapshot_result = match crate::failpoint::backend("checkpoint-snapshot") { + Ok(()) => backend.snapshot(snapshot).await, + Err(error) => Err(error), + }; + if let Err(error) = snapshot_result { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let flushed = match crate::failpoint::storage("checkpoint-storage-flush") { + Ok(()) => self.storage.flush_dirty(&storage).await, + Err(error) => Err(error), + }; + if let Err(error) = flushed { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let captured = match crate::failpoint::storage("checkpoint-rootfs-capture") { + Ok(()) => { + self.storage + .capture_checkpoint(&storage, &rootfs_path) + .await + } + Err(error) => Err(error), + }; + if let Err(error) = captured { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + + let parent = match self.checkpoints.read_head(id) { + Ok(parent) => parent, + Err(error) => { + return self + .finish_failed_unpublished_checkpoint( + id, + &backend, + &checkpoint_id, + checkpoint_store_error(error), + ) + .await; + } + }; + if let Err(error) = crate::failpoint::storage("checkpoint-publish") { + return self + .finish_failed_unpublished_checkpoint(id, &backend, &checkpoint_id, error.into()) + .await; + } + let published = self + .checkpoints + .publish( + &stage, + CommitCheckpoint { + parent, + policy_name: instance.policy_name.clone(), + image_digest: instance.image_digest.clone(), + backend: instance.backend, + backend_version, + snapshot_kind: SnapshotKind::Full, + }, + ) + .map_err(checkpoint_store_error); + let metadata = match published { + Ok(metadata) => metadata, + Err(error) => { + return self + .fail_published_checkpoint( + &backend, + &instance, + error, + "publication with uncertain outcome", + ) + .await; + } + }; + + if let Err(error) = instance.advance_checkpoint_phase(OperationPhase::CheckpointPublished) { + return self + .fail_published_checkpoint( + &backend, + &instance, + error.into(), + "published journal update", + ) + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-published-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .fail_published_checkpoint(&backend, &instance, error, "published state commit") + .await; + } + crate::failpoint::pause("checkpoint-after-publish-before-head").await; + + if let Err(error) = crate::failpoint::storage("checkpoint-head-update") { + return self + .finish_failed_published_checkpoint(id, &backend, error.into()) + .await; + } + if let Err(error) = self.checkpoints.set_head(id, &checkpoint_id) { + return self + .fail_published_checkpoint( + &backend, + &instance, + checkpoint_store_error(error), + "HEAD update with uncertain outcome", + ) + .await; + } + + if let Err(error) = instance.advance_checkpoint_phase(OperationPhase::CheckpointHeadUpdated) + { + return self + .fail_published_checkpoint(&backend, &instance, error.into(), "HEAD journal update") + .await; + } + if let Err(error) = crate::failpoint::state("checkpoint-head-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return self + .fail_published_checkpoint(&backend, &instance, error, "HEAD state commit") + .await; + } + crate::failpoint::pause("checkpoint-after-head").await; + + let resumed = match crate::failpoint::backend("checkpoint-resume") { + Ok(()) => backend.resume().await, + Err(error) => Err(error), + }; + if let Err(error) = resumed { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint {checkpoint_id} became HEAD, but backend resume failed: {error}" + ))); + } + if let Err(error) = self.verify_backend_ready(id, &backend).await { + self.mark_recovery(id)?; + return Err(error); + } + + if let Err(error) = instance + .transition(SandboxState::Checkpointed) + .and_then(|_| instance.transition(SandboxState::Running)) + { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint runtime resumed, but lifecycle transition failed: {error}" + ))); + } + instance.last_checkpoint = Some(checkpoint_id); + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("checkpoint-final-state") + .and_then(|_| self.persist_and_retain(instance)) + { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint completed, but final lifecycle state could not be committed: {error}" + ))); + } + drop(operation); + Ok(metadata) + } + + /// List every committed checkpoint and its HEAD reachability. + pub async fn list_checkpoints(&self, id: Uuid) -> Result> { + let _operation = self.operation_lock(id).lock_owned().await; + self.get(id)?; + self.checkpoints.list(id).map_err(checkpoint_store_error) + } + + async fn finish_failed_unpublished_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + checkpoint_id: &str, + original: BlazeDaemonError, + ) -> Result { + let compensation = self + .resume_and_clear_checkpoint(id, backend, Some(checkpoint_id)) + .await; + match compensation { + Ok(()) => Err(original), + Err(compensation) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{original}; checkpoint compensation failed: {compensation}" + ))), + } + } + + async fn finish_failed_published_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + original: BlazeDaemonError, + ) -> Result { + let compensation = self.resume_and_clear_checkpoint(id, backend, None).await; + match compensation { + Ok(()) => Err(original), + Err(compensation) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{original}; checkpoint compensation failed: {compensation}" + ))), + } + } + + async fn fail_published_checkpoint( + &self, + backend: &DynBackendInstance, + instance: &SandboxInstance, + original: BlazeDaemonError, + boundary: &str, + ) -> Result { + let resume = self.resume_backend(backend).await; + let recovery = self.mark_instance_recovery(instance.clone()); + Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint {boundary} failed: {original}{}{}", + resume + .err() + .map(|error| format!("; backend resume failed: {error}")) + .unwrap_or_default(), + recovery + .err() + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))) + } + + async fn resume_and_clear_checkpoint( + &self, + id: Uuid, + backend: &DynBackendInstance, + staging_checkpoint_id: Option<&str>, + ) -> Result<()> { + if let Err(error) = self.resume_backend(backend).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "backend resume failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Some(checkpoint_id) = staging_checkpoint_id + && let Err(error) = self.checkpoints.abort_staging(id, checkpoint_id) + { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint staging cleanup failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + + let mut instance = self.get(id)?; + if instance.state == SandboxState::Paused { + instance.transition(SandboxState::Running)?; + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "checkpoint compensation state commit failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + Ok(()) + } + + async fn resume_backend(&self, backend: &DynBackendInstance) -> Result<()> { + match crate::failpoint::backend("checkpoint-compensation-resume") { + Ok(()) => backend.resume().await?, + Err(error) => return Err(error.into()), + } + self.verify_backend_ready(backend.instance_id(), backend) + .await + } + + async fn require_live_backend(&self, id: Uuid, backend: &DynBackendInstance) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited before checkpoint capture \ + (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))) + } + Err(error) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))) + } + } + } + + async fn verify_backend_ready(&self, id: Uuid, backend: &DynBackendInstance) -> Result<()> { + self.require_live_backend(id, backend).await?; + self.wait_for_guest_ready(backend, "checkpoint-guest-ready") + .await?; + self.require_live_backend(id, backend).await + } +} + +fn checkpoint_store_error(error: impl std::fmt::Display) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("checkpoint store: {error}")) +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index f92384560a..72ec540dc4 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -18,6 +18,7 @@ use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; use tokio_util::sync::CancellationToken; use uuid::Uuid; +use crate::checkpoint_store::CheckpointStore; use crate::error::{BlazeDaemonError, Result}; use crate::guest::{GuestClient, GuestExecResult, MAX_GUEST_FILE_BYTES}; use crate::metrics::Metrics; @@ -80,8 +81,9 @@ pub struct SandboxManager { pool: Arc>, spawners: Arc, active_backend: BackendKind, - storage: Arc, + pub(super) storage: Arc, state_dir: PathBuf, + pub(super) checkpoints: CheckpointStore, rootfs_size: u64, mem_size: u64, metrics: Arc, @@ -125,6 +127,7 @@ impl SandboxManager { let backend_instances = Arc::new(Mutex::new(HashMap::new())); let pool = Arc::new(Mutex::new(pool)); let metrics = Arc::new(Metrics::new()); + let checkpoints = CheckpointStore::new(state_dir.join("checkpoints")); let resources = SandboxManagerResources { #[cfg(test)] instances: instances.clone(), @@ -141,6 +144,7 @@ impl SandboxManager { active_backend, storage, state_dir, + checkpoints, rootfs_size, mem_size, metrics, @@ -185,8 +189,7 @@ impl SandboxManager { Ok(operation) } - #[cfg(test)] - pub fn backend_owner(&self, id: Uuid) -> Option { + pub(crate) fn backend_owner(&self, id: Uuid) -> Option { match self.backend_instances.lock() { Ok(instances) => instances.get(&id).cloned(), Err(poisoned) => poisoned.into_inner().get(&id).cloned(), @@ -850,6 +853,17 @@ impl SandboxManager { ))); } + if let Err(error) = self.checkpoints.cleanup_transaction_artifacts(id) { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but checkpoint cleanup failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { let recovery = self.mark_recovery(id).err(); return Err(BlazeDaemonError::RecoveryRequired(format!( @@ -937,7 +951,7 @@ impl SandboxManager { )) } - async fn wait_for_guest_ready( + pub(super) async fn wait_for_guest_ready( &self, backend: &DynBackendInstance, failpoint: &str, @@ -1199,11 +1213,19 @@ impl SandboxManager { } } - fn mark_recovery(&self, id: Uuid) -> Result<()> { + pub(super) fn mark_recovery(&self, id: Uuid) -> Result<()> { self.mark_instance_recovery(self.get(id)?) } - fn mark_instance_recovery(&self, mut instance: SandboxInstance) -> Result<()> { + pub(super) fn persist_and_retain(&self, instance: SandboxInstance) -> Result<()> { + instance.persist(&self.state_dir)?; + if let Some(error) = self.retain_instance(instance) { + return Err(BlazeDaemonError::RecoveryRequired(error)); + } + Ok(()) + } + + pub(super) fn mark_instance_recovery(&self, mut instance: SandboxInstance) -> Result<()> { if instance.state == SandboxState::Destroyed { if instance.operation.as_ref().map(|operation| operation.kind) != Some(OperationKind::Destroy) @@ -1238,7 +1260,7 @@ impl SandboxManager { } } - fn retain_instance(&self, instance: SandboxInstance) -> Option { + pub(super) fn retain_instance(&self, instance: SandboxInstance) -> Option { match self.instances.lock() { Ok(mut instances) => { instances.insert(instance.id, instance); diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index e41ed62617..6d9c4583b4 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -14,7 +14,7 @@ use std::time::Duration; use std::time::Instant; use async_trait::async_trait; -use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::backend::{BackendKind, SnapshotRequest, SnapshotResult, SpawnRequest}; use blaze_core::guest_protocol::DEFAULT_MAX_RESPONSE_BYTES; use blaze_core::{BlazeError, Result}; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; @@ -46,8 +46,23 @@ pub struct SpawnResult { /// Owned runtime instance returned by a backend spawner. #[async_trait] pub trait BackendInstance: Send + Sync { + /// Stable sandbox identifier. + /// + /// The nil default prevents legacy or test-only owners from claiming a + /// real sandbox identity until they explicitly implement this contract. + fn instance_id(&self) -> Uuid { + Uuid::nil() + } /// Concrete backend implementation. fn backend(&self) -> BackendKind; + /// Backend version frozen into checkpoint metadata when available. + fn version(&self) -> Option<&str> { + None + } + /// Whether pause, resume, and full snapshot capture are implemented. + fn supports_checkpoint_capture(&self) -> bool { + false + } /// Guest transport endpoint, or an empty path for guestless backends. fn guest_socket_path(&self) -> &Path { Path::new("") @@ -58,6 +73,24 @@ pub trait BackendInstance: Send + Sync { /// Once an exit is observed, later calls continue to report a completed /// result even though the underlying handle has already been consumed. async fn try_wait(&self) -> Result>; + /// Pause guest execution for a consistent snapshot. + async fn pause(&self) -> Result<()> { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint pause", self.backend()), + }) + } + /// Resume guest execution after snapshot capture. + async fn resume(&self) -> Result<()> { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint resume", self.backend()), + }) + } + /// Write a self-contained snapshot. + async fn snapshot(&self, _request: SnapshotRequest) -> Result { + Err(BlazeError::BackendError { + msg: format!("{} does not support checkpoint capture", self.backend()), + }) + } /// Terminate the process and release all backend-owned resources. async fn kill(&self) -> Result<()>; } @@ -351,6 +384,7 @@ struct MockInstance { guest_socket_path: PathBuf, cancellation: CancellationToken, task: Mutex>>, + files: Arc>>>, killed: AtomicBool, } @@ -388,16 +422,29 @@ async fn spawn_mock_instance(instance_id: Uuid, run_dir: PathBuf) -> Result Uuid { + self.instance_id + } + fn backend(&self) -> BackendKind { BackendKind::Mock } + fn version(&self) -> Option<&str> { + Some("mock-v1") + } + + fn supports_checkpoint_capture(&self) -> bool { + true + } + fn guest_socket_path(&self) -> &Path { &self.guest_socket_path } @@ -427,6 +474,44 @@ impl BackendInstance for MockInstance { })) } + async fn pause(&self) -> Result<()> { + Ok(()) + } + + async fn resume(&self) -> Result<()> { + Ok(()) + } + + async fn snapshot(&self, request: SnapshotRequest) -> Result { + for path in [&request.snapshot_path, &request.mem_path] { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + tokio::fs::create_dir_all(parent).await?; + } + } + let vmstate = serde_json::to_vec(&serde_json::json!({ + "format": "blaze-mock-v1", + "instance_id": self.instance_id, + "kind": request.kind, + })) + .map_err(|error| BlazeError::BackendError { + msg: format!("serialize mock VM state: {error}"), + })?; + tokio::fs::write(&request.snapshot_path, vmstate).await?; + let memory = serde_json::to_vec(&*self.files.lock().await).map_err(|error| { + BlazeError::BackendError { + msg: format!("serialize mock guest state: {error}"), + } + })?; + tokio::fs::write(&request.mem_path, memory).await?; + Ok(SnapshotResult { + snapshot_path: request.snapshot_path, + mem_path: request.mem_path, + }) + } + async fn kill(&self) -> Result<()> { if self.killed.load(Ordering::Acquire) { return Ok(()); @@ -939,7 +1024,7 @@ mod tests { #[cfg(target_os = "linux")] use std::time::Duration; - use blaze_core::backend::SpawnRequest; + use blaze_core::backend::{SnapshotKind, SnapshotRequest, SpawnRequest}; use blaze_core::policy::BackendConfigs; use blaze_core::storage::StorageSlot; @@ -947,6 +1032,23 @@ mod tests { use super::*; + struct UnsupportedInstance; + + #[async_trait] + impl BackendInstance for UnsupportedInstance { + fn backend(&self) -> BackendKind { + BackendKind::Bubblewrap + } + + async fn try_wait(&self) -> Result> { + Ok(None) + } + + async fn kill(&self) -> Result<()> { + Ok(()) + } + } + fn request(root: &Path) -> SpawnRequest { let id = Uuid::new_v4(); let slot_dir = root.join("slot"); @@ -1015,6 +1117,70 @@ mod tests { instance.kill().await.expect("idempotent kill"); } + #[tokio::test] + async fn checkpoint_capture_defaults_fail_closed() { + let temp = tempfile::tempdir().expect("temp"); + let instance = UnsupportedInstance; + let request = SnapshotRequest { + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + kind: SnapshotKind::Full, + }; + + assert_eq!(instance.instance_id(), Uuid::nil()); + assert_eq!(instance.version(), None); + assert!(!instance.supports_checkpoint_capture()); + assert!(instance.pause().await.is_err()); + assert!(instance.resume().await.is_err()); + assert!(instance.snapshot(request).await.is_err()); + } + + #[tokio::test] + async fn mock_instance_captures_self_contained_state() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = request(temp.path()); + let instance_id = spawn.instance_id; + let instance = MockSpawner.spawn(spawn).await.expect("spawn"); + let client = GuestClient::new( + temp.path().join("run/vsock.uds"), + Duration::from_secs(1), + 1024, + ); + client + .write_file("/tmp/value".into(), b"captured") + .await + .expect("write guest state"); + let snapshot_path = temp.path().join("checkpoint/vmstate.snap"); + let mem_path = temp.path().join("checkpoint/memory.snap"); + + assert_eq!(instance.instance_id(), instance_id); + assert_eq!(instance.version(), Some("mock-v1")); + assert!(instance.supports_checkpoint_capture()); + instance.pause().await.expect("pause"); + let result = instance + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + kind: SnapshotKind::Full, + }) + .await + .expect("snapshot"); + instance.resume().await.expect("resume"); + + assert_eq!(result.snapshot_path, snapshot_path); + assert_eq!(result.mem_path, mem_path); + let vmstate: serde_json::Value = + serde_json::from_slice(&std::fs::read(&result.snapshot_path).expect("VM state")) + .expect("VM state JSON"); + assert_eq!(vmstate["instance_id"], instance_id.to_string()); + assert_eq!(vmstate["kind"], "full"); + let memory: HashMap> = + serde_json::from_slice(&std::fs::read(&result.mem_path).expect("memory")) + .expect("memory JSON"); + assert_eq!(memory.get("/tmp/value"), Some(&b"captured".to_vec())); + instance.kill().await.expect("kill"); + } + #[cfg(target_os = "linux")] #[tokio::test] async fn child_termination_requests_graceful_exit_first() { From 5804993623c936c36fe575a529bdf73a21dc8b18 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 09:48:41 +0800 Subject: [PATCH 28/40] feat(blaze): prune checkpoint history This enables operators to remove committed checkpoint branches that are not retained by HEAD or durable sandbox state through the checkpoint prune API. The catalog atomically renames each candidate to a tombstone before deletion. Retries, destroy, and startup reconciliation finish tombstones, trading immediate all-or-nothing deletion for durable incremental cleanup. The per-sandbox operation lock and unfinished-journal gate keep pruning from racing lifecycle operations. Checkpoint capture and backend-specific snapshot support are unchanged; checkpoint restore remains unavailable. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/AGENTS.md | 2 +- src/blaze/README.md | 9 + src/blaze/README_zh.md | 7 + src/blaze/crates/blazed/src/api.rs | 174 ++++++++++++++ .../crates/blazed/src/checkpoint_store.rs | 215 ++++++++++++++++-- .../crates/blazed/src/sandbox/checkpoint.rs | 21 +- .../crates/blazed/src/sandbox/manager.rs | 40 ++-- 7 files changed, 430 insertions(+), 38 deletions(-) diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index c93cca5ac0..d1ce1cb49d 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -30,7 +30,7 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. - **Lifecycle state machine**: The persisted model contains 9 states. Current - managed routes implement create, destroy, checkpoint capture/list, and + managed routes implement create, destroy, checkpoint capture/list/prune, and RecoveryRequired cleanup. Checkpoint capture returns `501` without advancing state when either the backend or storage provider lacks capture support. Reset remains reserved and returns `501` without advancing state. diff --git a/src/blaze/README.md b/src/blaze/README.md index cbd66a0729..8aa6c6e44f 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -140,6 +140,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/sandboxes/{id}/write` | Replace a guest file | | POST | `/v1/sandboxes/{id}/checkpoint` | Capture a full checkpoint when the backend and storage provider support it | | GET | `/v1/sandboxes/{id}/checkpoints` | List committed checkpoints and HEAD reachability | +| POST | `/v1/sandboxes/{id}/checkpoints/prune` | Remove checkpoint branches outside retained lineages | | GET | `/v1/instances` | Alias for listing sandboxes | | POST | `/v1/instances` | Alias for creating a sandbox | | GET | `/v1/instances/{id}` | Alias for sandbox details | @@ -150,6 +151,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances/{id}/write` | Compatible guest file write action | | POST | `/v1/instances/{id}/checkpoint` | Compatible full-checkpoint action | | GET | `/v1/instances/{id}/checkpoints` | Compatible checkpoint-list action | +| POST | `/v1/instances/{id}/checkpoints/prune` | Compatible checkpoint-prune action | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | @@ -210,6 +212,13 @@ remain available for explicit cleanup. Listing uses the same per-sandbox operation lock as capture, guest operations, and destroy. Destroy removes transaction scratch but preserves committed checkpoint history. +Checkpoint pruning removes committed entries outside the HEAD lineage and any +lineage still referenced by durable sandbox state. Each candidate is first +renamed to a hidden tombstone, so a retry, sandbox destroy, or startup +reconciliation can finish an interrupted deletion. Pruning uses the same +per-sandbox operation lock as capture and rejects an unfinished lifecycle +operation. + Runtime reset remains reserved and returns `501` without changing runtime or persisted state. diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index bf34d8d52f..231b1ba8fb 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -136,6 +136,7 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/sandboxes/{id}/write` | 替换 guest 文件 | | POST | `/v1/sandboxes/{id}/checkpoint` | 后端和存储 provider 支持时捕获完整 checkpoint | | GET | `/v1/sandboxes/{id}/checkpoints` | 列出已提交的 checkpoint 及其 HEAD 可达性 | +| POST | `/v1/sandboxes/{id}/checkpoints/prune` | 删除保留 lineage 之外的 checkpoint 分支 | | GET | `/v1/instances` | 列出 sandbox 的兼容入口 | | POST | `/v1/instances` | 创建 sandbox 的兼容入口 | | GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | @@ -146,6 +147,7 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | | POST | `/v1/instances/{id}/checkpoint` | 捕获完整 checkpoint 的兼容入口 | | GET | `/v1/instances/{id}/checkpoints` | 列出 checkpoint 的兼容入口 | +| POST | `/v1/instances/{id}/checkpoints/prune` | 删除 checkpoint 分支的兼容入口 | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | @@ -197,6 +199,11 @@ stage。如果发布或 HEAD 的结果无法确定，或者后端无法恢复， 保留，供后续显式清理。列出 checkpoint 与捕获、guest 操作及销毁共用同一个 sandbox 操作锁。销毁会删除事务临时文件，但保留已经提交的 checkpoint 历史。 +checkpoint pruning 会删除 HEAD lineage 和持久 sandbox 状态所引用 lineage 之外 +的已提交记录。每个待删除目录会先被重命名为隐藏 tombstone，因此重试、 +sandbox 销毁或启动恢复都可以完成被中断的删除。pruning 与 capture 共用 +sandbox 操作锁，并拒绝存在未完成生命周期操作的 sandbox。 + runtime reset 仍是预留接口，会返回 `501`，且不会修改 runtime 或持久化状态。 ### Guest 操作 diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index a7bb54fe02..1b7e3058bb 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -122,6 +122,10 @@ async fn dispatch( | ("POST", ["v1", "sandboxes", id, "checkpoint"]) => checkpoint(state, id).await, ("GET", ["v1", "instances", id, "checkpoints"]) | ("GET", ["v1", "sandboxes", id, "checkpoints"]) => list_checkpoints(state, id).await, + ("POST", ["v1", "instances", id, "checkpoints", "prune"]) + | ("POST", ["v1", "sandboxes", id, "checkpoints", "prune"]) => { + prune_checkpoints(state, id).await + } ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, ("DELETE", ["v1", "instances", id]) | ("DELETE", ["v1", "sandboxes", id]) @@ -312,6 +316,14 @@ async fn list_checkpoints(state: &Arc, id: &str) -> Result, id: &str) -> Result>> { + let removed = state.manager.prune_checkpoints(parse_uuid(id)?).await?; + json_ok(&json!({ + "removed": removed, + "count": removed.len(), + })) +} + async fn reset_instance(state: &Arc, id: &str) -> Result>> { let uuid = parse_uuid(id)?; let _operation = state @@ -2203,6 +2215,168 @@ mod tests { } } + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn startup_cleans_a_terminal_checkpoint_prune_tombstone() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + state.manager.destroy(uuid).await.expect("destroy sandbox"); + + let prune_hook = + crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + prune_hook + .run(state.manager.prune_checkpoints(uuid)) + .await + .expect_err("prune must stop after the durable tombstone"); + let checkpoint_dir = state.state_dir.join("checkpoints").join(&id); + assert!( + std::fs::read_dir(&checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + drop(state); + + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 1); + assert_eq!(report.completed, 1); + assert!(report.failures.is_empty()); + assert!( + !std::fs::read_dir(&checkpoint_dir) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + assert!( + restarted + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn checkpoint_prune_removes_an_unreachable_publication() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + write_checkpoint_fixture(&state, id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + let checkpoint_id = state + .manager + .list_checkpoints(uuid) + .await + .expect("unreachable checkpoint") + .pop() + .expect("checkpoint") + .id; + + let (status, pruned) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoints/prune"), + Vec::new(), + ) + .await; + + assert_eq!(status, StatusCode::OK); + assert_eq!(pruned["count"], 1); + assert_eq!(pruned["removed"], json!([checkpoint_id])); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn prune_retry_cleans_a_prior_checkpoint_tombstone() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let head_hook = crate::failpoint::TestFailpoint::new(&["checkpoint-head-update"]); + head_hook + .run(state.manager.checkpoint(uuid)) + .await + .expect_err("checkpoint must remain published but unreachable"); + let prune_hook = + crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + prune_hook + .run(state.manager.prune_checkpoints(uuid)) + .await + .expect_err("first prune must stop after the durable tombstone"); + + assert!( + state + .manager + .prune_checkpoints(uuid) + .await + .expect("retry prune") + .is_empty() + ); + assert!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("empty checkpoint catalog") + .is_empty() + ); + assert!( + !std::fs::read_dir(state.state_dir.join("checkpoints").join(&id)) + .expect("checkpoint catalog") + .filter_map(std::result::Result::ok) + .any(|entry| entry.file_name().to_string_lossy().starts_with(".prune.")) + ); + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn checkpoint_resume_failure_keeps_head_and_runtime_ownership() { diff --git a/src/blaze/crates/blazed/src/checkpoint_store.rs b/src/blaze/crates/blazed/src/checkpoint_store.rs index fb0ec265c2..1cbf40f746 100644 --- a/src/blaze/crates/blazed/src/checkpoint_store.rs +++ b/src/blaze/crates/blazed/src/checkpoint_store.rs @@ -3,7 +3,7 @@ //! //! Publication and HEAD updates are separate durability boundaries. A //! checkpoint can therefore be published but unreachable after an interrupted -//! operation; listing exposes that state for explicit inspection. +//! operation; listing exposes that state and pruning can remove it later. use std::collections::{HashMap, HashSet}; use std::fs::{self, File}; @@ -24,6 +24,7 @@ const METADATA_FILE: &str = "metadata.json"; const HEAD_FILE: &str = "HEAD"; const STAGING_SUFFIX: &str = ".tmp"; const TOMBSTONE_SUFFIX: &str = ".tombstone"; +const PRUNE_TOMBSTONE_PREFIX: &str = ".prune."; const ABORT_TOMBSTONE_PREFIX: &str = ".abort."; /// Failure while reading or mutating the daemon checkpoint catalog. @@ -274,6 +275,49 @@ impl CheckpointStore { Ok(checkpoints) } + /// Prune unreferenced branches while retaining HEAD and explicit lineages. + /// + /// Every candidate is first atomically renamed to a hidden tombstone and + /// the sandbox directory is synced. An interrupted delete is therefore + /// absent from the live catalog and can be completed by + /// [`Self::cleanup_transaction_artifacts`]. + pub fn prune_preserving(&self, sandbox_id: Uuid, protected: &[String]) -> Result> { + let catalog = self.load_catalog(sandbox_id)?; + let mut keep = HashSet::new(); + if let Some(head) = self.read_head(sandbox_id)? { + keep.extend(lineage_from(&catalog, &head)?); + } + for checkpoint_id in protected { + validate_checkpoint_id(checkpoint_id)?; + keep.extend(lineage_from(&catalog, checkpoint_id)?); + } + let mut candidates = catalog + .keys() + .filter(|checkpoint_id| !keep.contains(*checkpoint_id)) + .cloned() + .collect::>(); + candidates.sort(); + if candidates.is_empty() { + return Ok(Vec::new()); + } + let sandbox_dir = self + .optional_sandbox_dir(sandbox_id)? + .ok_or_else(|| invariant(format!("checkpoint sandbox {sandbox_id} disappeared")))?; + + let mut removed = Vec::with_capacity(candidates.len()); + for checkpoint_id in candidates { + let checkpoint_dir = self.committed_dir(sandbox_id, &checkpoint_id)?; + let tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, &checkpoint_id); + rename_path(&checkpoint_dir, &tombstone, "tombstone pruned checkpoint")?; + sync_directory(&sandbox_dir)?; + checkpoint_store_failpoint("checkpoint-prune-after-tombstone", &tombstone)?; + remove_directory(&tombstone, "remove pruned checkpoint tombstone")?; + sync_directory(&sandbox_dir)?; + removed.push(checkpoint_id); + } + Ok(removed) + } + /// Atomically move HEAD to an already committed, verified checkpoint. pub fn set_head(&self, sandbox_id: Uuid, checkpoint_id: &str) -> Result<()> { self.verify(sandbox_id, checkpoint_id)?; @@ -328,6 +372,16 @@ impl CheckpointStore { Ok(Some(checkpoint_id.to_string())) } + /// Return whether a checkpoint transaction left cleanup artifacts. + /// + /// Missing sandbox directories report `false`. Recognized names with an + /// unexpected file type return an error instead of being silently ignored, + /// matching [`Self::cleanup_transaction_artifacts`] so the caller can + /// report the unsafe layout without deleting an unrelated entry. + pub(crate) fn has_transaction_artifacts(&self, sandbox_id: Uuid) -> Result { + Ok(!self.transaction_artifacts(sandbox_id)?.is_empty()) + } + /// Remove incomplete stages, temporary HEAD files, and cleanup tombstones. /// /// Committed checkpoint directories and the published HEAD are retained. @@ -632,16 +686,18 @@ fn classify_scratch_name(name: &str) -> Result> { parse_uuid_component(nonce, "temporary HEAD")?; return Ok(Some(ScratchKind::File)); } - if let Some(body) = name - .strip_prefix(ABORT_TOMBSTONE_PREFIX) - .and_then(|name| name.strip_suffix(TOMBSTONE_SUFFIX)) - { - let (checkpoint_id, nonce) = body - .rsplit_once('.') - .ok_or_else(|| invariant(format!("invalid checkpoint tombstone {name:?}")))?; - validate_checkpoint_id(checkpoint_id)?; - parse_uuid_component(nonce, "checkpoint tombstone")?; - return Ok(Some(ScratchKind::Directory)); + for prefix in [PRUNE_TOMBSTONE_PREFIX, ABORT_TOMBSTONE_PREFIX] { + if let Some(body) = name + .strip_prefix(prefix) + .and_then(|name| name.strip_suffix(TOMBSTONE_SUFFIX)) + { + let (checkpoint_id, nonce) = body + .rsplit_once('.') + .ok_or_else(|| invariant(format!("invalid checkpoint tombstone {name:?}")))?; + validate_checkpoint_id(checkpoint_id)?; + parse_uuid_component(nonce, "checkpoint tombstone")?; + return Ok(Some(ScratchKind::Directory)); + } } Ok(None) } @@ -969,6 +1025,39 @@ mod tests { .expect("abort stage"); } + #[test] + fn prune_retains_head_and_explicit_lineages() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let protected = publish(&store, sandbox_id, Some(root.clone()), false); + let abandoned = publish(&store, sandbox_id, Some(root.clone()), false); + + let removed = store + .prune_preserving(sandbox_id, std::slice::from_ref(&protected)) + .expect("prune checkpoints"); + + assert_eq!(removed, vec![abandoned]); + assert!(store.verify(sandbox_id, &root).is_ok()); + assert!(store.verify(sandbox_id, &protected).is_ok()); + } + + #[test] + fn prune_can_remove_all_unreachable_checkpoints_before_head_exists() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let unreachable = publish(&store, sandbox_id, None, false); + + let removed = store + .prune_preserving(sandbox_id, &[]) + .expect("prune unreachable checkpoint"); + + assert_eq!(removed, vec![unreachable]); + assert!(store.list(sandbox_id).expect("list checkpoints").is_empty()); + } + #[test] fn cleanup_removes_transaction_scratch_but_retains_committed_history() { let temp = tempfile::tempdir().expect("tempdir"); @@ -979,8 +1068,8 @@ mod tests { let sandbox_dir = store.root.join(sandbox_id.to_string()); let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); - let tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, &committed); - fs::create_dir(&tombstone).expect("create abort tombstone"); + let tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, &committed); + fs::create_dir(&tombstone).expect("create prune tombstone"); let removed = store .cleanup_transaction_artifacts(sandbox_id) @@ -995,28 +1084,48 @@ mod tests { } #[test] - fn cleanup_recognizes_every_capture_transaction_artifact() { + fn transaction_artifact_detection_matches_cleanup_boundaries() { let temp = tempfile::tempdir().expect("tempdir"); let store = CheckpointStore::new(temp.path().join("checkpoints")); + let missing_sandbox = Uuid::new_v4(); + assert!( + !store + .has_transaction_artifacts(missing_sandbox) + .expect("inspect missing sandbox") + ); + let sandbox_id = Uuid::new_v4(); let stage = store.begin(sandbox_id).expect("begin checkpoint"); let sandbox_dir = store.root.join(sandbox_id.to_string()); let temporary_head = sandbox_dir.join(format!(".HEAD.{}{STAGING_SUFFIX}", Uuid::new_v4())); fs::write(&temporary_head, b"temporary").expect("write temporary HEAD"); + let prune_tombstone = tombstone_path(&sandbox_dir, PRUNE_TOMBSTONE_PREFIX, stage.id()); + fs::create_dir(&prune_tombstone).expect("create prune tombstone"); let abort_tombstone = tombstone_path(&sandbox_dir, ABORT_TOMBSTONE_PREFIX, stage.id()); fs::create_dir(&abort_tombstone).expect("create abort tombstone"); + assert!( + store + .has_transaction_artifacts(sandbox_id) + .expect("inspect transaction artifacts") + ); let removed = store .cleanup_transaction_artifacts(sandbox_id) .expect("cleanup transaction artifacts"); - assert_eq!(removed.len(), 3); + assert_eq!(removed.len(), 4); assert!(!stage.path.exists()); assert!(!temporary_head.exists()); + assert!(!prune_tombstone.exists()); assert!(!abort_tombstone.exists()); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect cleaned sandbox") + ); } #[test] - fn cleanup_reports_unsafe_layout_without_deleting_it() { + fn transaction_artifact_detection_reports_unsafe_layout_without_deleting_it() { let temp = tempfile::tempdir().expect("tempdir"); let store = CheckpointStore::new(temp.path().join("checkpoints")); let sandbox_id = Uuid::new_v4(); @@ -1024,6 +1133,10 @@ mod tests { fs::remove_dir(&stage.path).expect("remove staging directory"); fs::write(&stage.path, b"not a directory").expect("replace stage with file"); + let inspect_error = store + .has_transaction_artifacts(sandbox_id) + .expect_err("unsafe layout must be reported"); + assert!(inspect_error.to_string().contains("unexpected file type")); let cleanup_error = store .cleanup_transaction_artifacts(sandbox_id) .expect_err("cleanup must not delete an entry with the wrong type"); @@ -1055,6 +1168,12 @@ mod tests { let sandbox_id = Uuid::new_v4(); assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); assert!(store.list(sandbox_id).expect("list").is_empty()); + assert!( + store + .prune_preserving(sandbox_id, &[]) + .expect("prune") + .is_empty() + ); assert!( store .cleanup_transaction_artifacts(sandbox_id) @@ -1107,7 +1226,14 @@ mod tests { let store = CheckpointStore::new(temp.path().join("checkpoints")); let sandbox_id = Uuid::new_v4(); let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + assert_eq!( + store + .prune_preserving(sandbox_id, &[]) + .expect("prune unreachable checkpoint"), + vec![unreachable] + ); assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); } @@ -1140,10 +1266,9 @@ mod tests { .expect("renamed checkpoint remains committed"); assert_eq!(store.read_head(sandbox_id).expect("HEAD"), None); assert!( - store - .cleanup_transaction_artifacts(sandbox_id) + !store + .has_transaction_artifacts(sandbox_id) .expect("inspect transaction artifacts") - .is_empty() ); } @@ -1171,10 +1296,56 @@ mod tests { Some(checkpoint_id) ); assert!( - store - .cleanup_transaction_artifacts(sandbox_id) + !store + .has_transaction_artifacts(sandbox_id) .expect("inspect transaction artifacts") - .is_empty() ); } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn prune_boundary_error_leaves_a_cleanup_tombstone() { + let temp = tempfile::tempdir().expect("tempdir"); + let store = CheckpointStore::new(temp.path().join("checkpoints")); + let sandbox_id = Uuid::new_v4(); + let root = publish(&store, sandbox_id, None, true); + let unreachable = publish(&store, sandbox_id, Some(root.clone()), false); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-prune-after-tombstone"]); + + let error = hook + .run(async { store.prune_preserving(sandbox_id, &[]) }) + .await + .expect_err("prune boundary must return a store error"); + + assert!( + error + .to_string() + .contains("checkpoint-prune-after-tombstone") + ); + assert!(store.verify(sandbox_id, &unreachable).is_err()); + assert_eq!( + store + .list(sandbox_id) + .expect("list checkpoints after tombstone") + .into_iter() + .map(|checkpoint| checkpoint.id) + .collect::>(), + vec![root.clone()] + ); + assert!( + store + .has_transaction_artifacts(sandbox_id) + .expect("inspect prune tombstone") + ); + let removed = store + .cleanup_transaction_artifacts(sandbox_id) + .expect("remove prune tombstone"); + assert_eq!(removed.len(), 1); + assert!( + !store + .has_transaction_artifacts(sandbox_id) + .expect("inspect cleaned sandbox") + ); + assert_eq!(store.read_head(sandbox_id).expect("HEAD"), Some(root)); + } } diff --git a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs index cf8476a085..19400cbfc3 100644 --- a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs +++ b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs @@ -1,5 +1,5 @@ // SPDX-License-Identifier: Apache-2.0 -//! Durable checkpoint capture and listing. +//! Durable checkpoint capture, listing, and pruning. use blaze_core::backend::{SnapshotKind, SnapshotRequest}; use blaze_core::checkpoint::{CheckpointInfo, CheckpointMetadata, CommitCheckpoint}; @@ -301,6 +301,25 @@ impl SandboxManager { self.checkpoints.list(id).map_err(checkpoint_store_error) } + /// Remove branches not retained by HEAD or durable lifecycle references. + pub async fn prune_checkpoints(&self, id: Uuid) -> Result> { + let _operation = self.operation_lock(id).lock_owned().await; + let instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + self.checkpoints + .cleanup_transaction_artifacts(id) + .map_err(checkpoint_store_error)?; + let protected = instance.last_checkpoint.into_iter().collect::>(); + self.checkpoints + .prune_preserving(id, &protected) + .map_err(checkpoint_store_error) + } + async fn finish_failed_unpublished_checkpoint( &self, id: Uuid, diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 72ec540dc4..419856ccb9 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -61,9 +61,9 @@ pub struct ReconcileFailure { /// Aggregate startup cleanup outcome. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ReconcileReport { - /// Number of records that were not clean terminal states. + /// Number of records selected for runtime or transaction cleanup. pub attempted: usize, - /// Number moved to the terminal state. + /// Number of selected records whose cleanup completed. pub completed: usize, /// Records that remain recoverable. pub failures: Vec, @@ -762,6 +762,16 @@ impl SandboxManager { .map_err(|_| poisoned("backend_instances"))? .contains_key(&id); if is_clean_terminal(&original) && !has_retained_backend { + if let Err(error) = self.checkpoints.cleanup_transaction_artifacts(id) { + original.begin_destroy_recovery(); + let recovery = self.persist_and_retain(original).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: checkpoint transaction cleanup failed: {error}{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } return Ok(false); } @@ -975,19 +985,21 @@ impl SandboxManager { &self, item_timeout: Duration, ) -> ReconcileReport { - let mut ids = match self.instances.lock() { - Ok(instances) => instances - .values() - .filter(|instance| !is_clean_terminal(instance)) - .map(|instance| instance.id) - .collect::>(), - Err(poisoned) => poisoned - .into_inner() - .values() - .filter(|instance| !is_clean_terminal(instance)) - .map(|instance| instance.id) - .collect::>(), + let records = match self.instances.lock() { + Ok(instances) => instances.values().cloned().collect::>(), + Err(poisoned) => poisoned.into_inner().values().cloned().collect::>(), }; + let mut ids = records + .into_iter() + .filter(|instance| { + !is_clean_terminal(instance) + || self + .checkpoints + .has_transaction_artifacts(instance.id) + .unwrap_or(true) + }) + .map(|instance| instance.id) + .collect::>(); match self.backend_instances.lock() { Ok(instances) => ids.extend(instances.keys().copied()), Err(poisoned) => ids.extend(poisoned.into_inner().keys().copied()), From 0df7f3ee91d43d6f6292a45712f9db937ab42ddf Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 09:54:18 +0800 Subject: [PATCH 29/40] feat(blaze): capture Firecracker state This enables Firecracker-backed sandboxes to use the existing checkpoint API by pausing the VM, writing full VM and memory snapshots through the Firecracker UDS API, and resuming the owned runtime. The spawner resolves the configured binary version on every start and freezes it into the runtime owner, so concurrent probes and binary replacement cannot reuse stale global provenance. Firecracker records require that version before capture begins, and the UDS client bounds response size and request duration. Only full capture is enabled; restore and incremental snapshots remain unavailable. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/checkpoint.rs | 44 ++ .../crates/blazed/src/sandbox/checkpoint.rs | 3 +- .../crates/blazed/src/spawner/firecracker.rs | 553 +++++++++++++++++- 3 files changed, 577 insertions(+), 23 deletions(-) diff --git a/src/blaze/crates/blaze-core/src/checkpoint.rs b/src/blaze/crates/blaze-core/src/checkpoint.rs index e7ccc4d01a..22eb299b73 100644 --- a/src/blaze/crates/blaze-core/src/checkpoint.rs +++ b/src/blaze/crates/blaze-core/src/checkpoint.rs @@ -182,6 +182,7 @@ pub fn validate_commit_checkpoint( checkpoint_id, &input.policy_name, &input.image_digest, + input.backend, input.backend_version.as_deref(), ) } @@ -234,6 +235,7 @@ pub fn validate_checkpoint_manifest( &metadata.id, &metadata.policy_name, &metadata.image_digest, + metadata.backend, metadata.backend_version.as_deref(), )?; validate_artifact_manifest(&metadata.id, &metadata.artifacts) @@ -243,6 +245,7 @@ fn validate_runtime_identity( checkpoint_id: &str, policy_name: &str, image_digest: &str, + backend: BackendKind, backend_version: Option<&str>, ) -> Result<(), CheckpointValidationError> { if policy_name.trim().is_empty() { @@ -266,6 +269,13 @@ fn validate_runtime_identity( reason: "present version is empty".to_string(), }); } + if backend == BackendKind::Firecracker && backend_version.is_none() { + return Err(CheckpointValidationError::InvalidField { + checkpoint_id: checkpoint_id.to_string(), + field: "backend_version", + reason: "Firecracker captures require a backend version".to_string(), + }); + } Ok(()) } @@ -440,4 +450,38 @@ mod tests { }; assert!(validate_commit_checkpoint(&id, &input).is_err()); } + + #[test] + fn firecracker_checkpoint_records_require_a_backend_version() { + let mut metadata = metadata(); + metadata.backend = BackendKind::Firecracker; + metadata.backend_version = None; + let error = validate_checkpoint_manifest(&metadata, metadata.sandbox_id, &metadata.id) + .expect_err("missing version must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidField { + field: "backend_version", + .. + } + )); + + let input = CommitCheckpoint { + parent: metadata.parent, + policy_name: metadata.policy_name, + image_digest: metadata.image_digest, + backend: metadata.backend, + backend_version: metadata.backend_version, + snapshot_kind: metadata.snapshot_kind, + }; + let error = validate_commit_checkpoint(&metadata.id, &input) + .expect_err("missing version must fail"); + assert!(matches!( + error, + CheckpointValidationError::InvalidField { + field: "backend_version", + .. + } + )); + } } diff --git a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs index cf8476a085..42505c0708 100644 --- a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs +++ b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 //! Durable checkpoint capture and listing. -use blaze_core::backend::{SnapshotKind, SnapshotRequest}; +use blaze_core::backend::{BackendKind, SnapshotKind, SnapshotRequest}; use blaze_core::checkpoint::{CheckpointInfo, CheckpointMetadata, CommitCheckpoint}; use blaze_core::lifecycle::{OperationPhase, SandboxInstance, SandboxState}; use uuid::Uuid; @@ -48,6 +48,7 @@ impl SandboxManager { if backend_version .as_deref() .is_some_and(|version| version.trim().is_empty()) + || (backend.backend() == BackendKind::Firecracker && backend_version.is_none()) { return Err(BlazeDaemonError::UnsupportedOperation(format!( "instance {id} backend {} does not report a usable checkpoint version", diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index cb25dc9302..7d30b0240b 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -8,9 +8,16 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use async_trait::async_trait; -use blaze_core::backend::{BackendKind, SpawnRequest}; +use blaze_core::backend::{ + BackendKind, SnapshotKind, SnapshotRequest, SnapshotResult, SpawnRequest, +}; use blaze_core::policy::{FirecrackerConfig, VmConfig, parse_memory_value, to_mib_ceil}; use blaze_core::{BlazeError, Result}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Bytes; +use hyper::client::conn::http1; +use hyper::{Method, Request}; +use hyper_util::rt::TokioIo; use tokio::net::UnixStream; use tokio::process::{Child, Command}; use tokio::sync::Mutex; @@ -24,11 +31,13 @@ use super::{ spawn_result, stopped_marker, terminate_child, }; +const MAX_API_RESPONSE_BYTES: usize = 64 * 1024; + /// Firecracker backend factory. pub struct FirecrackerSpawner { images_dir: PathBuf, + api_timeout: Duration, socket_timeout: Duration, - version: Mutex>, } impl FirecrackerSpawner { @@ -36,11 +45,24 @@ impl FirecrackerSpawner { pub fn new(images_dir: PathBuf) -> Self { Self { images_dir, + api_timeout: Duration::from_secs(30), socket_timeout: Duration::from_secs(5), - version: Mutex::new(None), } } + async fn capture_for( + &self, + binary_path: &Path, + api_socket: PathBuf, + ) -> Result { + let backend_version = read_backend_version(binary_path).await?; + Ok(FirecrackerCapture::new( + api_socket, + self.api_timeout, + backend_version, + )) + } + async fn start( &self, request: SpawnRequest, @@ -50,6 +72,9 @@ impl FirecrackerSpawner { validate_regular_file(&self.images_dir.join("vmlinux"), "vmlinux")?; tokio::fs::create_dir_all(&request.run_dir).await?; let api_socket = request.run_dir.join("api.sock"); + let capture = self + .capture_for(&request.binary_path, api_socket.clone()) + .await?; let guest_socket = request.run_dir.join("vsock.uds"); let pid_file = request.run_dir.join("firecracker.pid"); let stopped_marker = stopped_marker(&request.run_dir); @@ -78,7 +103,7 @@ impl FirecrackerSpawner { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( request.instance_id, child, - api_socket, + capture, guest_socket, fc_config.enable_vsock, pid_file, @@ -90,7 +115,7 @@ impl FirecrackerSpawner { let instance = FirecrackerInstance::new( request.instance_id, child, - api_socket, + capture, guest_socket, fc_config.enable_vsock, pid_file, @@ -119,10 +144,7 @@ impl BackendSpawner for FirecrackerSpawner { return Ok(false); } match read_backend_version(binary_path).await { - Ok(version) => { - *self.version.lock().await = Some(version); - Ok(true) - } + Ok(_) => Ok(true), Err(error) => { tracing::debug!(%error, binary = %binary_path.display(), "firecracker version probe failed"); Ok(false) @@ -138,7 +160,7 @@ impl BackendSpawner for FirecrackerSpawner { struct FirecrackerInstance { instance_id: Uuid, child: Mutex>, - api_socket: PathBuf, + capture: FirecrackerCapture, guest_socket: PathBuf, pid_file: PathBuf, stopped_marker: PathBuf, @@ -149,7 +171,7 @@ impl FirecrackerInstance { fn new( instance_id: Uuid, child: Child, - api_socket: PathBuf, + capture: FirecrackerCapture, guest_socket: PathBuf, enable_vsock: bool, pid_file: PathBuf, @@ -158,7 +180,7 @@ impl FirecrackerInstance { Self { instance_id, child: Mutex::new(Some(child)), - api_socket, + capture, guest_socket: configured_guest_socket(enable_vsock, guest_socket), pid_file, stopped_marker, @@ -169,10 +191,22 @@ impl FirecrackerInstance { #[async_trait] impl BackendInstance for FirecrackerInstance { + fn instance_id(&self) -> Uuid { + self.instance_id + } + fn backend(&self) -> BackendKind { BackendKind::Firecracker } + fn version(&self) -> Option<&str> { + Some(&self.capture.backend_version) + } + + fn supports_checkpoint_capture(&self) -> bool { + true + } + fn guest_socket_path(&self) -> &Path { &self.guest_socket } @@ -198,6 +232,62 @@ impl BackendInstance for FirecrackerInstance { Ok(Some(spawn_result(self.instance_id, status))) } + async fn pause(&self) -> Result<()> { + self.capture + .api + .call_json( + Method::PATCH, + "/vm", + Some(serde_json::json!({"state": "Paused"})), + ) + .await?; + Ok(()) + } + + async fn resume(&self) -> Result<()> { + self.capture + .api + .call_json( + Method::PATCH, + "/vm", + Some(serde_json::json!({"state": "Resumed"})), + ) + .await?; + Ok(()) + } + + async fn snapshot(&self, request: SnapshotRequest) -> Result { + let SnapshotRequest { + snapshot_path, + mem_path, + kind: SnapshotKind::Full, + } = request; + for path in [&snapshot_path, &mem_path] { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + tokio::fs::create_dir_all(parent).await?; + } + } + self.capture + .api + .call_json( + Method::PUT, + "/snapshot/create", + Some(serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_file_path": mem_path, + "snapshot_type": "Full" + })), + ) + .await?; + Ok(SnapshotResult { + snapshot_path, + mem_path, + }) + } + async fn kill(&self) -> Result<()> { if self.killed.load(Ordering::Acquire) { return Ok(()); @@ -218,9 +308,112 @@ impl BackendInstance for FirecrackerInstance { } } +struct FirecrackerCapture { + api: FirecrackerApiClient, + backend_version: String, +} + +impl FirecrackerCapture { + fn new(api_socket: PathBuf, api_timeout: Duration, backend_version: String) -> Self { + Self { + api: FirecrackerApiClient::new(api_socket, api_timeout), + backend_version, + } + } +} + +#[derive(Debug, Clone)] +struct FirecrackerApiClient { + socket: PathBuf, + timeout: Duration, +} + +impl FirecrackerApiClient { + fn new(socket: PathBuf, timeout: Duration) -> Self { + Self { socket, timeout } + } + + async fn call_json( + &self, + method: Method, + path: &str, + body: Option, + ) -> Result> { + let operation = async { + let stream = UnixStream::connect(&self.socket).await?; + let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) + .await + .map_err(backend_protocol_error)?; + tokio::spawn(async move { + if let Err(error) = connection.await { + tracing::debug!(%error, "firecracker API connection ended"); + } + }); + let bytes = match body { + Some(body) => { + serde_json::to_vec(&body).map_err(|error| BlazeError::BackendError { + msg: format!("serialize Firecracker API request: {error}"), + })? + } + None => Vec::new(), + }; + let mut builder = Request::builder() + .method(method.clone()) + .uri(format!("http://localhost{path}")); + if !bytes.is_empty() { + builder = builder.header("content-type", "application/json"); + } + let request = builder + .body(Full::new(Bytes::from(bytes))) + .map_err(|error| BlazeError::BackendError { + msg: format!("build Firecracker API request: {error}"), + })?; + let response = sender + .send_request(request) + .await + .map_err(backend_protocol_error)?; + let status = response.status(); + let mut response_body = response.into_body(); + let mut collected = Vec::new(); + while let Some(frame) = response_body.frame().await { + let frame = frame.map_err(backend_protocol_error)?; + if let Ok(data) = frame.into_data() { + let remaining = MAX_API_RESPONSE_BYTES.saturating_sub(collected.len()); + collected.extend_from_slice(&data[..data.len().min(remaining)]); + if data.len() > remaining { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} response exceeded \ + {MAX_API_RESPONSE_BYTES} bytes" + ), + }); + } + } + } + if !status.is_success() { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} returned {status}: {}", + String::from_utf8_lossy(&collected) + ), + }); + } + Ok(collected) + }; + tokio::time::timeout(self.timeout, operation) + .await + .map_err(|_| BlazeError::BackendError { + msg: format!( + "Firecracker {method} {path} timed out after {:?}", + self.timeout + ), + })? + } +} + impl FirecrackerInstance { async fn cleanup(&self) -> Result<()> { - remove_if_exists(&self.api_socket).await?; + remove_if_exists(&self.capture.api.socket).await?; remove_if_exists(&self.guest_socket).await?; remove_if_exists(&self.pid_file).await?; Ok(()) @@ -325,14 +518,29 @@ fn configure_logs(command: &mut Command, run_dir: &Path, serial_log: bool) -> Re } async fn read_backend_version(binary_path: &Path) -> Result { - let output = tokio::time::timeout( - Duration::from_secs(5), - Command::new(binary_path).arg("--version").output(), - ) - .await - .map_err(|_| BlazeError::BackendError { - msg: format!("firecracker probe timed out: {}", binary_path.display()), - })??; + let mut busy_retries = 0; + let output = loop { + match tokio::time::timeout( + Duration::from_secs(5), + Command::new(binary_path).arg("--version").output(), + ) + .await + { + Ok(Ok(output)) => break output, + Ok(Err(error)) + if error.kind() == std::io::ErrorKind::ExecutableFileBusy && busy_retries < 3 => + { + busy_retries += 1; + tokio::time::sleep(Duration::from_millis(5)).await; + } + Ok(Err(error)) => return Err(error.into()), + Err(_) => { + return Err(BlazeError::BackendError { + msg: format!("firecracker probe timed out: {}", binary_path.display()), + }); + } + } + }; if !output.status.success() { return Err(BlazeError::BackendError { msg: format!( @@ -471,6 +679,12 @@ fn path_string<'a>(path: &'a Path, label: &str) -> Result<&'a str> { }) } +fn backend_protocol_error(error: hyper::Error) -> BlazeError { + BlazeError::BackendError { + msg: format!("Firecracker API protocol error: {error}"), + } +} + pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> Result<()> { let stopped_marker = stopped_marker(run_dir); if stopped_marker.is_file() { @@ -503,7 +717,22 @@ pub(super) async fn cleanup_orphan_run_dir(instance_id: Uuid, run_dir: &Path) -> #[cfg(test)] mod tests { + #[cfg(target_os = "linux")] + use std::convert::Infallible; + use blaze_core::storage::StorageSlot; + #[cfg(target_os = "linux")] + use http_body_util::BodyExt; + #[cfg(target_os = "linux")] + use hyper::Response; + #[cfg(target_os = "linux")] + use hyper::server::conn::http1 as server_http1; + #[cfg(target_os = "linux")] + use hyper::service::service_fn; + #[cfg(target_os = "linux")] + use tokio::net::UnixListener; + #[cfg(target_os = "linux")] + use tokio::sync::oneshot; use super::*; @@ -524,6 +753,282 @@ mod tests { assert!(parse_backend_version(b"Firecracker v1.15.0\nFirecracker v1.16.0\n").is_err()); } + #[cfg(target_os = "linux")] + async fn spawn_api( + socket: &Path, + call_count: usize, + ) -> oneshot::Receiver> { + let listener = UnixListener::bind(socket).expect("bind"); + let (tx, rx) = oneshot::channel(); + tokio::spawn(async move { + let observed = Arc::new(Mutex::new(Vec::with_capacity(call_count))); + for _ in 0..call_count { + let (stream, _) = listener.accept().await.expect("accept"); + let observed = observed.clone(); + let service = service_fn(move |request: Request| { + let observed = observed.clone(); + async move { + let method = request.method().clone(); + let path = request.uri().path().to_string(); + let body = request + .into_body() + .collect() + .await + .expect("request body") + .to_bytes(); + let body = if body.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_slice(&body).expect("request JSON") + }; + observed.lock().await.push((method, path, body)); + Ok::<_, Infallible>( + Response::builder() + .status(hyper::StatusCode::NO_CONTENT) + .body(Full::new(Bytes::new())) + .expect("response"), + ) + } + }); + server_http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await + .expect("serve"); + } + let calls = observed.lock().await.clone(); + let _ = tx.send(calls); + }); + rx + } + + #[cfg(target_os = "linux")] + fn spawn_api_response( + socket: &Path, + status: hyper::StatusCode, + body: Vec, + delay: Duration, + ) -> tokio::task::JoinHandle<()> { + let listener = UnixListener::bind(socket).expect("bind"); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.expect("accept"); + let body = Bytes::from(body); + let service = service_fn(move |_request: Request| { + let body = body.clone(); + async move { + if !delay.is_zero() { + tokio::time::sleep(delay).await; + } + Ok::<_, Infallible>( + Response::builder() + .status(status) + .body(Full::new(body)) + .expect("response"), + ) + } + }); + let _ = server_http1::Builder::new() + .serve_connection(TokioIo::new(stream), service) + .await; + }) + } + + #[cfg(target_os = "linux")] + fn write_version_binary(path: &Path, output: &str) { + use std::os::unix::fs::PermissionsExt; + + let staged = path.with_extension("new"); + std::fs::write(&staged, format!("#!/bin/sh\nprintf '%s\\n' '{output}'\n")) + .expect("write version binary"); + std::fs::set_permissions(&staged, std::fs::Permissions::from_mode(0o755)) + .expect("make version binary executable"); + std::fs::rename(staged, path).expect("replace version binary"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn launch_capture_reads_the_requested_binary_each_time() { + let temp = tempfile::tempdir().expect("temp"); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + write_version_binary(&first, "Firecracker v1.15.0"); + write_version_binary(&second, "Firecracker v1.16.0"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + let first_capture = spawner + .capture_for(&first, temp.path().join("first.sock")) + .await + .expect("first capture"); + let second_capture = spawner + .capture_for(&second, temp.path().join("second.sock")) + .await + .expect("second capture"); + assert_eq!(first_capture.backend_version, "Firecracker v1.15.0"); + assert_eq!(second_capture.backend_version, "Firecracker v1.16.0"); + + write_version_binary(&first, "Firecracker v1.17.0"); + let replaced_capture = spawner + .capture_for(&first, temp.path().join("replaced.sock")) + .await + .expect("replaced capture"); + assert_eq!(replaced_capture.backend_version, "Firecracker v1.17.0"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn probe_checks_each_requested_binary() { + let temp = tempfile::tempdir().expect("temp"); + let valid = temp.path().join("valid"); + let invalid = temp.path().join("invalid"); + write_version_binary(&valid, "Firecracker v1.16.0"); + write_version_binary(&invalid, "not a Firecracker version"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + assert!(spawner.probe(&valid).await.expect("valid probe")); + assert!(!spawner.probe(&invalid).await.expect("invalid probe")); + + write_version_binary(&invalid, "Firecracker v1.17.0"); + assert!(spawner.probe(&invalid).await.expect("replaced probe")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn instance_reports_version_and_captures_full_snapshot_over_uds() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 3).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let instance_id = Uuid::new_v4(); + let instance = FirecrackerInstance::new( + instance_id, + child, + FirecrackerCapture::new( + api_socket, + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + PathBuf::new(), + false, + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + ); + let snapshot_path = temp.path().join("checkpoint/vmstate.snap"); + let mem_path = temp.path().join("checkpoint/memory.snap"); + + assert_eq!(instance.instance_id(), instance_id); + assert_eq!(instance.version(), Some("Firecracker v1.16.0")); + assert!(instance.supports_checkpoint_capture()); + instance.pause().await.expect("pause"); + let result = instance + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + kind: SnapshotKind::Full, + }) + .await + .expect("snapshot"); + instance.resume().await.expect("resume"); + + assert_eq!(result.snapshot_path, snapshot_path); + assert_eq!(result.mem_path, mem_path); + let calls = observed.await.expect("observed calls"); + assert_eq!( + calls, + vec![ + ( + Method::PATCH, + "/vm".to_string(), + serde_json::json!({"state": "Paused"}), + ), + ( + Method::PUT, + "/snapshot/create".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_file_path": mem_path, + "snapshot_type": "Full", + }), + ), + ( + Method::PATCH, + "/vm".to_string(), + serde_json::json!({"state": "Resumed"}), + ), + ] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_reports_non_success_response_body() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::BAD_REQUEST, + b"invalid VM state".to_vec(), + Duration::ZERO, + ); + let client = FirecrackerApiClient::new(socket, Duration::from_secs(1)); + + let error = client + .call_json(Method::PATCH, "/vm", None) + .await + .expect_err("non-success response"); + server.await.expect("server"); + + let message = error.to_string(); + assert!(message.contains("400 Bad Request")); + assert!(message.contains("invalid VM state")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_rejects_an_oversized_response() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::OK, + vec![b'x'; MAX_API_RESPONSE_BYTES + 1], + Duration::ZERO, + ); + let client = FirecrackerApiClient::new(socket, Duration::from_secs(1)); + + let error = client + .call_json(Method::GET, "/vm", None) + .await + .expect_err("oversized response"); + server.await.expect("server"); + + assert!(error.to_string().contains("response exceeded 65536 bytes")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn api_client_times_out_a_stalled_response() { + let temp = tempfile::tempdir().expect("temp"); + let socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &socket, + hyper::StatusCode::OK, + Vec::new(), + Duration::from_millis(200), + ); + let client = FirecrackerApiClient::new(socket, Duration::from_millis(20)); + + let error = client + .call_json(Method::GET, "/vm", None) + .await + .expect_err("stalled response"); + server.await.expect("server"); + + assert!(error.to_string().contains("timed out after 20ms")); + } + #[test] fn vm_config_omits_network_until_the_network_capability_is_enabled() { let temp = tempfile::tempdir().expect("temp"); @@ -632,7 +1137,11 @@ mod tests { let owner: DynBackendInstance = Arc::new(FirecrackerInstance::new( Uuid::new_v4(), child, - temp.path().join("api.sock"), + FirecrackerCapture::new( + temp.path().join("api.sock"), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), temp.path().join("guest.sock"), true, pid_file.clone(), From b0d2d7d584a804cd0aad66388d06b2c23f438508 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 10:53:49 +0800 Subject: [PATCH 30/40] feat(blaze): journal checkpoint restores This provides blaze-core with a durable restore operation and monotonic ownership phases for replacing a running sandbox from a selected checkpoint. A later sandbox-manager commit consumes these records; this commit does not expose restore through a production entry point. Previously the lifecycle model could journal capture progress but could not distinguish a staged rootfs, a stopped backend, an activated replacement, or a committed storage transaction during restore. Treating every interruption as the same state would hide whether compensation is still possible. The Restoring state is entered only after the old backend is stopped and may return to Running only after storage commit. The journal keeps the selected target separate from last_checkpoint. That field records only the last completed capture; restore never changes it, while HEAD records the selection. Storage, backend, manager, and API implementations remain out of scope. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/AGENTS.md | 2 +- src/blaze/crates/blaze-core/src/lifecycle.rs | 304 ++++++++++++++++++- 2 files changed, 296 insertions(+), 10 deletions(-) diff --git a/src/blaze/AGENTS.md b/src/blaze/AGENTS.md index c93cca5ac0..f6e5efefef 100644 --- a/src/blaze/AGENTS.md +++ b/src/blaze/AGENTS.md @@ -29,7 +29,7 @@ Platform: Linux (x86_64 + aarch64) for production. macOS builds succeed but spaw - **Daemon-only API model**: No CLI client for sandbox operations. All instance/pool/template management is done via HTTP endpoints on UDS (`/run/blaze/api.sock`) or TCP (`:14159`). The CLI subcommands (`daemon start`, `daemon reload`, `daemon doctor`) only manage daemon lifecycle. - **BackendSpawner trait**: All backend-specific process management is behind `BackendSpawner`. Adding a new backend means implementing `spawn()`, `wait()`, `kill()`, `probe()` and registering it in `daemon::build_spawner()`. - **Policy-driven backend selection**: Workload class → policy file → prioritized backend list. The daemon probes backends at startup and selects the first available. Never hardcode backend preference in application logic. -- **Lifecycle state machine**: The persisted model contains 9 states. Current +- **Lifecycle state machine**: The persisted model contains 10 states. Current managed routes implement create, destroy, checkpoint capture/list, and RecoveryRequired cleanup. Checkpoint capture returns `501` without advancing state when either the backend or storage provider lacks capture support. diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index 735bc7db66..e2753a6224 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -22,6 +22,8 @@ pub enum SandboxState { Running, Paused, Checkpointed, + /// The previous backend is stopped while replacement resources are owned. + Restoring, RecoveryRequired, Reset, Warm, @@ -36,6 +38,7 @@ impl SandboxState { SandboxState::Running => "running", SandboxState::Paused => "paused", SandboxState::Checkpointed => "checkpointed", + SandboxState::Restoring => "restoring", SandboxState::RecoveryRequired => "recovery-required", SandboxState::Reset => "reset", SandboxState::Warm => "warm", @@ -52,6 +55,8 @@ pub enum OperationKind { Create, /// A point-in-time checkpoint is being captured and published. Checkpoint, + /// A running sandbox is being replaced from a selected checkpoint. + Restore, /// Runtime resources are being destroyed. Destroy, } @@ -61,6 +66,7 @@ impl OperationKind { match self { OperationKind::Create => "create", OperationKind::Checkpoint => "checkpoint", + OperationKind::Restore => "restore", OperationKind::Destroy => "destroy", } } @@ -75,8 +81,8 @@ impl std::fmt::Display for OperationKind { /// Durable boundary reached by a multi-step lifecycle operation. /// /// The journal keeps this separate from [`SandboxState`]: state describes -/// externally visible runtime availability, while the phase identifies which -/// checkpoint resources may already have been published after interruption. +/// externally visible runtime availability, while the phase identifies the +/// last resource-ownership or catalog boundary committed before interruption. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum OperationPhase { @@ -88,6 +94,20 @@ pub enum OperationPhase { CheckpointPublished, /// HEAD references the checkpoint; runtime resume is not yet committed. CheckpointHeadUpdated, + /// Restore intent is durable, but the current runtime is still owned. + RestorePreparing, + /// Replacement storage is staged without changing the live rootfs. + RestoreStorageStaged, + /// The current backend has been confirmed stopped. + RestoreBackendStopped, + /// Staged storage is active while the predecessor remains recoverable. + RestoreStorageActivated, + /// A replacement backend has started and is owned by the runtime. + RestoreBackendStarted, + /// HEAD references the restored checkpoint; storage and lifecycle commits remain. + RestoreHeadUpdated, + /// The storage replacement is committed and can no longer be aborted. + RestoreStorageCommitted, } impl OperationPhase { @@ -97,6 +117,29 @@ impl OperationPhase { OperationPhase::CheckpointPaused => "checkpoint-paused", OperationPhase::CheckpointPublished => "checkpoint-published", OperationPhase::CheckpointHeadUpdated => "checkpoint-head-updated", + OperationPhase::RestorePreparing => "restore-preparing", + OperationPhase::RestoreStorageStaged => "restore-storage-staged", + OperationPhase::RestoreBackendStopped => "restore-backend-stopped", + OperationPhase::RestoreStorageActivated => "restore-storage-activated", + OperationPhase::RestoreBackendStarted => "restore-backend-started", + OperationPhase::RestoreHeadUpdated => "restore-head-updated", + OperationPhase::RestoreStorageCommitted => "restore-storage-committed", + } + } + + const fn operation_kind(self) -> OperationKind { + match self { + OperationPhase::CheckpointPreparing + | OperationPhase::CheckpointPaused + | OperationPhase::CheckpointPublished + | OperationPhase::CheckpointHeadUpdated => OperationKind::Checkpoint, + OperationPhase::RestorePreparing + | OperationPhase::RestoreStorageStaged + | OperationPhase::RestoreBackendStopped + | OperationPhase::RestoreStorageActivated + | OperationPhase::RestoreBackendStarted + | OperationPhase::RestoreHeadUpdated + | OperationPhase::RestoreStorageCommitted => OperationKind::Restore, } } @@ -106,6 +149,13 @@ impl OperationPhase { OperationPhase::CheckpointPaused => 1, OperationPhase::CheckpointPublished => 2, OperationPhase::CheckpointHeadUpdated => 3, + OperationPhase::RestorePreparing => 0, + OperationPhase::RestoreStorageStaged => 1, + OperationPhase::RestoreBackendStopped => 2, + OperationPhase::RestoreStorageActivated => 3, + OperationPhase::RestoreBackendStarted => 4, + OperationPhase::RestoreHeadUpdated => 5, + OperationPhase::RestoreStorageCommitted => 6, } } } @@ -240,21 +290,53 @@ impl SandboxInstance { /// Advance the active checkpoint journal without replacing its identity. pub fn advance_checkpoint_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Checkpoint, phase) + } + + /// Record restore intent without changing the last completed checkpoint. + pub fn begin_restore_operation(&mut self, checkpoint_id: String) -> Result<()> { + self.begin_operation(OperationKind::Restore)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.checkpoint_id = Some(checkpoint_id); + operation.phase = Some(OperationPhase::RestorePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active restore journal without replacing its identity. + pub fn advance_restore_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Restore, phase) + } + + fn advance_operation_phase( + &mut self, + requested_kind: OperationKind, + phase: OperationPhase, + ) -> Result<()> { + if phase.operation_kind() != requested_kind { + return Err(BlazeError::InvalidStateTransition { + from: requested_kind.to_string(), + to: phase.as_str().to_string(), + }); + } let operation = self .operation .as_mut() .ok_or_else(|| BlazeError::OperationInProgress { active: "none".to_string(), - requested: OperationKind::Checkpoint.to_string(), + requested: requested_kind.to_string(), })?; - if operation.kind != OperationKind::Checkpoint { + if operation.kind != requested_kind { return Err(BlazeError::OperationInProgress { active: operation.kind.to_string(), - requested: OperationKind::Checkpoint.to_string(), + requested: requested_kind.to_string(), }); } if let Some(current) = operation.phase - && phase.rank() < current.rank() + && (current.operation_kind() != requested_kind || phase.rank() < current.rank()) { return Err(BlazeError::InvalidStateTransition { from: current.as_str().to_string(), @@ -289,11 +371,24 @@ impl SandboxInstance { self.updated_at = Utc::now(); } - /// Apply a state transition. Returns + /// Apply a state transition. + /// + /// Restore transitions additionally require the durable backend-stop and + /// storage-commit boundaries before changing externally visible state. + /// Returns /// [`BlazeError::InvalidStateTransition`] when the move is not part /// of the lifecycle state graph. pub fn transition(&mut self, target: SandboxState) -> Result<()> { - if !is_valid_transition(self.state, target) { + let restore_boundary_reached = match (self.state, target) { + (_, SandboxState::Restoring) => { + self.restore_phase_reached(OperationPhase::RestoreBackendStopped) + } + (SandboxState::Restoring, SandboxState::Running) => { + self.restore_phase_reached(OperationPhase::RestoreStorageCommitted) + } + _ => true, + }; + if !restore_boundary_reached || !is_valid_transition(self.state, target) { return Err(BlazeError::InvalidStateTransition { from: self.state.to_string(), to: target.to_string(), @@ -322,6 +417,17 @@ impl SandboxInstance { Ok(()) } + fn restore_phase_reached(&self, minimum: OperationPhase) -> bool { + minimum.operation_kind() == OperationKind::Restore + && self.operation.as_ref().is_some_and(|operation| { + operation.kind == OperationKind::Restore + && operation.phase.is_some_and(|phase| { + phase.operation_kind() == OperationKind::Restore + && phase.rank() >= minimum.rank() + }) + }) + } + /// Persist this instance to `{state_dir}/{id}/state.json`. Atomic /// rename via `state.json.tmp` to avoid torn reads on daemon restart. pub fn persist(&self, state_dir: &Path) -> Result<()> { @@ -414,7 +520,8 @@ impl SandboxInstance { fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { use SandboxState::{ - Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Reset, Running, Warm, + Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Reset, Restoring, + Running, Warm, }; if to == Destroyed { // `* → destroyed` is always valid (terminal sink). @@ -431,6 +538,8 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { (Paused, Checkpointed) => true, (Paused, Running) => true, // resume (Checkpointed, Running) => true, + (Running, Restoring) => true, + (Restoring, Running) => true, (Reset, Warm) => true, (Warm, Creating) => true, // pool reuse / warm path _ => false, @@ -481,6 +590,49 @@ mod tests { assert_eq!(inst.start_path, StartPath::Warm); } + #[test] + fn restore_state_requires_owned_replacement_boundaries() { + let mut inst = fresh(); + let error = inst + .transition(SandboxState::Restoring) + .expect_err("pending sandbox cannot restore"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Pending); + + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect("begin restore"); + inst.advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect("stage storage"); + let error = inst + .transition(SandboxState::Restoring) + .expect_err("running remains visible until the backend is stopped"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Running); + + inst.advance_restore_phase(OperationPhase::RestoreBackendStopped) + .expect("stop backend"); + inst.transition(SandboxState::Restoring) + .expect("restore starts"); + inst.advance_restore_phase(OperationPhase::RestoreStorageActivated) + .expect("activate storage"); + inst.advance_restore_phase(OperationPhase::RestoreBackendStarted) + .expect("start backend"); + inst.advance_restore_phase(OperationPhase::RestoreHeadUpdated) + .expect("update head"); + let error = inst + .transition(SandboxState::Running) + .expect_err("storage commit precedes the final running state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!(inst.state, SandboxState::Restoring); + + inst.advance_restore_phase(OperationPhase::RestoreStorageCommitted) + .expect("commit storage"); + inst.transition(SandboxState::Running) + .expect("restore commits"); + } + #[test] fn destroy_is_always_legal_except_from_destroyed() { let mut inst = fresh(); @@ -781,4 +933,138 @@ mod tests { )); assert_eq!(instance.operation, journal); } + + #[test] + fn restore_journal_round_trips_without_overwriting_last_checkpoint() { + let tmp = tempfile::tempdir().expect("tmp"); + let mut instance = fresh(); + let completed = "ckpt-00000000-0000-0000-0000-000000000001".to_string(); + let selected = "ckpt-00000000-0000-0000-0000-000000000002".to_string(); + instance.last_checkpoint = Some(completed.clone()); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance + .begin_restore_operation(selected.clone()) + .expect("begin restore"); + instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect("stage storage"); + instance + .advance_restore_phase(OperationPhase::RestoreBackendStopped) + .expect("stop backend"); + instance + .transition(SandboxState::Restoring) + .expect("restoring"); + + for phase in [ + OperationPhase::RestoreStorageActivated, + OperationPhase::RestoreBackendStarted, + OperationPhase::RestoreHeadUpdated, + OperationPhase::RestoreStorageCommitted, + ] { + instance + .advance_restore_phase(phase) + .expect("advance restore"); + assert_eq!( + instance.last_checkpoint.as_deref(), + Some(completed.as_str()) + ); + } + instance.persist(tmp.path()).expect("persist"); + + let loaded = SandboxInstance::load(tmp.path(), instance.id).expect("load"); + let journal = loaded.operation.expect("restore journal"); + assert_eq!(journal.kind, OperationKind::Restore); + assert_eq!(journal.checkpoint_id.as_deref(), Some(selected.as_str())); + assert_eq!(journal.phase, Some(OperationPhase::RestoreStorageCommitted)); + assert_eq!(loaded.last_checkpoint.as_deref(), Some(completed.as_str())); + assert_eq!( + serde_json::to_value(journal.kind).expect("serialize kind"), + serde_json::json!("restore") + ); + assert_eq!( + serde_json::to_value(journal.phase).expect("serialize phase"), + serde_json::json!("restore-storage-committed") + ); + } + + #[test] + fn restore_journal_rejects_phase_regression() { + let mut instance = fresh(); + let completed = "ckpt-00000000-0000-0000-0000-000000000001".to_string(); + instance.last_checkpoint = Some(completed.clone()); + instance + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000002".to_string()) + .expect("begin restore"); + instance + .advance_restore_phase(OperationPhase::RestoreStorageActivated) + .expect("advance restore"); + + let error = instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .expect_err("restore phase must remain a durable lower bound"); + + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + assert_eq!( + instance + .operation + .as_ref() + .and_then(|journal| journal.phase), + Some(OperationPhase::RestoreStorageActivated) + ); + assert_eq!(instance.last_checkpoint, Some(completed)); + } + + #[test] + fn operation_journals_reject_phases_from_the_other_operation() { + let mut checkpoint = fresh(); + checkpoint + .begin_checkpoint_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect("begin checkpoint"); + let checkpoint_journal = checkpoint.operation.clone(); + let checkpoint_error = checkpoint + .advance_checkpoint_phase(OperationPhase::RestoreBackendStopped) + .expect_err("checkpoint cannot record restore progress"); + assert!(matches!( + checkpoint_error, + BlazeError::InvalidStateTransition { .. } + )); + assert_eq!(checkpoint.operation, checkpoint_journal); + + let mut restore = fresh(); + restore + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000002".to_string()) + .expect("begin restore"); + let restore_journal = restore.operation.clone(); + let restore_error = restore + .advance_restore_phase(OperationPhase::CheckpointPublished) + .expect_err("restore cannot record checkpoint progress"); + assert!(matches!( + restore_error, + BlazeError::InvalidStateTransition { .. } + )); + assert_eq!(restore.operation, restore_journal); + } + + #[test] + fn restore_journal_cannot_replace_an_active_operation() { + let mut instance = fresh(); + instance + .begin_operation(OperationKind::Create) + .expect("begin create"); + let journal = instance.operation.clone(); + + let error = instance + .begin_restore_operation("ckpt-00000000-0000-0000-0000-000000000001".to_string()) + .expect_err("restore must not replace create"); + + assert!(matches!( + error, + BlazeError::OperationInProgress { active, requested } + if active == "create" && requested == "restore" + )); + assert_eq!(instance.operation, journal); + } } From a1799086d5d5d9c17a9cc9d07501e3fa3014b6f2 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 10:58:20 +0800 Subject: [PATCH 31/40] feat(blaze): stage recoverable rootfs restores This enables FileStorageProvider to prepare a checkpoint rootfs through opt-in StorageProvider restore transactions and later select that staged copy. Staging copies the checkpoint without changing the live rootfs. Activation uses same-directory renames to retain the original inode. Durable states let commit, abort, or restart reconciliation finish each interrupted boundary. This trades temporary disk capacity for deterministic rollback. The provider contract is fail-closed by default. Lifecycle orchestration, backend restore, daemon routes, and user-facing documentation remain unwired. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/storage.rs | 80 + src/blaze/crates/blazed/src/file_provider.rs | 42 +- .../blazed/src/file_provider/restore.rs | 1541 +++++++++++++++++ 3 files changed, 1662 insertions(+), 1 deletion(-) create mode 100644 src/blaze/crates/blazed/src/file_provider/restore.rs diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index 8bc5c00d15..9423b4d6d9 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -32,6 +32,19 @@ pub struct StorageSlot { pub instance_dir: PathBuf, } +/// Stable handle for one provider-owned rootfs restore transaction. +/// +/// Callers must keep this handle from staging through activation and +/// finalization. Providers must validate both fields against durable state +/// before changing storage. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageRestoreTransaction { + /// Stable sandbox identifier whose rootfs is being replaced. + pub instance_id: String, + /// Unique transaction identifier used to reject stale handles. + pub transaction_id: uuid::Uuid, +} + /// Pool readiness status. #[derive(Debug, Clone, Default, serde::Serialize)] pub struct PoolStatus { @@ -144,9 +157,76 @@ pub trait StorageProvider: Send + Sync { }) } + /// Report whether this provider can restore a self-contained checkpoint. + /// + /// The default is conservative so existing providers cannot enter a + /// partially implemented replacement flow. + fn supports_checkpoint_restore(&self) -> bool { + false + } + + /// Copy a checkpoint rootfs into provider-owned staging storage. + /// + /// Staging must leave the live rootfs unchanged so callers may prepare the + /// replacement before stopping the current runtime. + async fn stage_checkpoint_restore( + &self, + slot: &StorageSlot, + source: &Path, + ) -> Result { + let _ = (slot, source); + Err(checkpoint_restore_unsupported()) + } + + /// Select the staged rootfs while retaining the previous rootfs. + /// + /// A successful activation must remain abortable until + /// [`Self::commit_checkpoint_restore`] starts. + async fn activate_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Finalize an activated rootfs and release its retained predecessor. + async fn commit_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Restore the predecessor retained by a staged or activated transaction. + async fn abort_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + let _ = transaction; + Err(checkpoint_restore_unsupported()) + } + + /// Resolve an interrupted restore transaction after process restart. + /// + /// Implementations choose the outcome from durable transaction state: + /// work not yet committed should roll back, while a durable commit intent + /// should finish committing. + async fn reconcile_checkpoint_restore(&self, instance_id: &str) -> Result<()> { + let _ = instance_id; + Err(checkpoint_restore_unsupported()) + } + /// Query warm pool status. fn pool_status(&self) -> PoolStatus; /// Drain all ready slots from the warm pool. async fn drain_pool(&self) -> Result; } + +fn checkpoint_restore_unsupported() -> BlazeError { + BlazeError::StorageError { + msg: "storage provider does not support checkpoint restore".to_string(), + } +} diff --git a/src/blaze/crates/blazed/src/file_provider.rs b/src/blaze/crates/blazed/src/file_provider.rs index 84fafce120..d6d7e638c3 100644 --- a/src/blaze/crates/blazed/src/file_provider.rs +++ b/src/blaze/crates/blazed/src/file_provider.rs @@ -11,9 +11,12 @@ use uuid::Uuid; use blaze_core::error::{BlazeError, Result}; use blaze_core::storage::{ - AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageRestoreTransaction, + StorageSlot, }; +mod restore; + /// A filesystem-based provider that copies base artifacts when available and /// otherwise creates sparse rootfs and memory files at configured sizes. pub struct FileStorageProvider { @@ -352,6 +355,43 @@ impl StorageProvider for FileStorageProvider { }) } + fn supports_checkpoint_restore(&self) -> bool { + true + } + + async fn stage_checkpoint_restore( + &self, + slot: &StorageSlot, + source: &Path, + ) -> Result { + restore::stage(self, slot, source).await + } + + async fn activate_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::activate(self, transaction).await + } + + async fn commit_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::commit(self, transaction).await + } + + async fn abort_checkpoint_restore( + &self, + transaction: &StorageRestoreTransaction, + ) -> Result<()> { + restore::abort(self, transaction).await + } + + async fn reconcile_checkpoint_restore(&self, instance_id: &str) -> Result<()> { + restore::reconcile(self, instance_id).await + } + fn pool_status(&self) -> PoolStatus { PoolStatus::default() } diff --git a/src/blaze/crates/blazed/src/file_provider/restore.rs b/src/blaze/crates/blazed/src/file_provider/restore.rs new file mode 100644 index 0000000000..fcb0c72452 --- /dev/null +++ b/src/blaze/crates/blazed/src/file_provider/restore.rs @@ -0,0 +1,1541 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable rootfs replacement for the file storage provider. + +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use uuid::Uuid; + +use blaze_core::error::{BlazeError, Result}; +use blaze_core::storage::{StorageRestoreTransaction, StorageSlot}; + +use super::{FileStorageProvider, RequiredPathType}; + +const JOURNAL_VERSION: u32 = 1; +const MAX_JOURNAL_SIZE: u64 = 16 * 1024; +const COPY_BUFFER_SIZE: usize = 64 * 1024; + +#[derive(Debug)] +struct RestorePaths { + instance_id: String, + instance_dir: PathBuf, + rootfs: PathBuf, + copying: PathBuf, + staged: PathBuf, + backup: PathBuf, + discard: PathBuf, + journal: PathBuf, + journal_temporary: PathBuf, +} + +impl RestorePaths { + fn transaction_artifacts(&self) -> [&Path; 6] { + [ + &self.copying, + &self.staged, + &self.backup, + &self.discard, + &self.journal, + &self.journal_temporary, + ] + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +enum RestoreState { + Staged, + Activated, + Aborting, + Committing, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct RestoreJournal { + version: u32, + instance_id: String, + transaction_id: Uuid, + state: RestoreState, +} + +impl RestoreJournal { + fn transaction(&self) -> StorageRestoreTransaction { + StorageRestoreTransaction { + instance_id: self.instance_id.clone(), + transaction_id: self.transaction_id, + } + } +} + +/// Removes files that have not yet become part of a durable transaction. +struct UnpublishedFiles { + paths: Vec, +} + +impl UnpublishedFiles { + fn new() -> Self { + Self { paths: Vec::new() } + } + + fn track(&mut self, path: &Path) { + self.paths.push(path.to_path_buf()); + } + + fn untrack(&mut self, path: &Path) { + self.paths.retain(|tracked| tracked != path); + } + + fn commit(&mut self) { + self.paths.clear(); + } +} + +impl Drop for UnpublishedFiles { + fn drop(&mut self) { + for path in self.paths.drain(..) { + let _ = std::fs::remove_file(path); + } + } +} + +pub(super) async fn stage( + provider: &FileStorageProvider, + slot: &StorageSlot, + source: &Path, +) -> Result { + let paths = restore_paths(provider, &slot.id).await?; + require_plain_file(&paths.rootfs, "live rootfs").await?; + ensure_no_transaction(&paths).await?; + + let source = canonical_plain_file(source, "restore source").await?; + let rootfs = tokio::fs::canonicalize(&paths.rootfs) + .await + .map_err(|error| storage_error(format!("canonicalize live rootfs: {error}")))?; + if source == rootfs { + return Err(storage_error( + "restore source must be independent from the live rootfs", + )); + } + + let journal = RestoreJournal { + version: JOURNAL_VERSION, + instance_id: paths.instance_id.clone(), + transaction_id: Uuid::new_v4(), + state: RestoreState::Staged, + }; + let mut unpublished = UnpublishedFiles::new(); + + copy_for_restore(&source, &paths.copying, &mut unpublished).await?; + rename_new_plain_file(&paths.copying, &paths.staged).await?; + unpublished.untrack(&paths.copying); + unpublished.track(&paths.staged); + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-stage").await; + + publish_new_journal(&paths, &journal, &mut unpublished).await?; + unpublished.commit(); + sync_directory(&paths.instance_dir).await?; + Ok(journal.transaction()) +} + +pub(super) async fn activate( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let mut journal = require_journal(&paths).await?; + verify_transaction(&journal, transaction)?; + + match journal.state { + RestoreState::Activated => return ensure_activated_layout(&paths).await, + RestoreState::Staged => {} + RestoreState::Aborting => { + return Err(storage_error(format!( + "restore transaction {} is aborting", + transaction.transaction_id + ))); + } + RestoreState::Committing => { + return Err(storage_error(format!( + "restore transaction {} is committing", + transaction.transaction_id + ))); + } + } + + let (live, staged, backup, discard) = inspect_layout(&paths).await?; + if discard { + return Err(invalid_layout(&paths, journal.state)); + } + + if live && staged && !backup { + rename_new_plain_file(&paths.rootfs, &paths.backup).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-backup").await; + } else if !live && staged && backup { + // Resume after the predecessor was retained. + } else if live && !staged && backup { + // Resume after the staged rootfs was selected. + } else { + return Err(invalid_layout(&paths, journal.state)); + } + + let (live, staged, backup, discard) = inspect_layout(&paths).await?; + if !live && staged && backup && !discard { + if let Err(selection) = select_staged_rootfs(&paths).await { + let rollback = match crate::failpoint::storage("storage-restore-switch-rollback") { + Ok(()) => match rename_new_plain_file(&paths.backup, &paths.rootfs).await { + Ok(()) => sync_directory(&paths.instance_dir).await, + Err(error) => Err(error), + }, + Err(error) => Err(error), + }; + return match rollback { + Ok(()) => Err(storage_error(format!( + "select staged rootfs for '{}': {selection}; predecessor restored", + paths.instance_id + ))), + Err(rollback) => Err(storage_error(format!( + "select staged rootfs for '{}': {selection}; restoring predecessor failed: \ + {rollback}", + paths.instance_id + ))), + }; + } + crate::failpoint::pause("storage-restore-after-switch").await; + } + + ensure_activated_layout(&paths).await?; + journal.state = RestoreState::Activated; + replace_journal(&paths, &journal).await +} + +async fn select_staged_rootfs(paths: &RestorePaths) -> Result<()> { + crate::failpoint::storage("storage-restore-switch")?; + rename_new_plain_file(&paths.staged, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await +} + +pub(super) async fn commit( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let Some(mut journal) = read_journal(&paths).await? else { + return ensure_finalized_layout(&paths).await; + }; + verify_transaction(&journal, transaction)?; + + match journal.state { + RestoreState::Activated => { + ensure_activated_layout(&paths).await?; + journal.state = RestoreState::Committing; + replace_journal(&paths, &journal).await?; + crate::failpoint::pause("storage-restore-after-commit-intent").await; + } + RestoreState::Committing => {} + RestoreState::Staged => { + return Err(storage_error(format!( + "restore transaction {} is not activated", + transaction.transaction_id + ))); + } + RestoreState::Aborting => { + return Err(storage_error(format!( + "restore transaction {} is aborting", + transaction.transaction_id + ))); + } + } + finish_commit(&paths).await +} + +pub(super) async fn abort( + provider: &FileStorageProvider, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + let paths = restore_paths(provider, &transaction.instance_id).await?; + ensure_no_transient_files(&paths).await?; + let Some(mut journal) = read_journal(&paths).await? else { + return ensure_finalized_layout(&paths).await; + }; + verify_transaction(&journal, transaction)?; + + if journal.state == RestoreState::Committing { + return Err(storage_error(format!( + "restore transaction {} has durable commit intent", + transaction.transaction_id + ))); + } + if journal.state != RestoreState::Aborting { + journal.state = RestoreState::Aborting; + replace_journal(&paths, &journal).await?; + } + finish_abort(&paths).await +} + +pub(super) async fn reconcile(provider: &FileStorageProvider, instance_id: &str) -> Result<()> { + let paths = restore_paths(provider, instance_id).await?; + remove_plain_file_if_present(&paths.copying, "restore copying file").await?; + remove_plain_file_if_present(&paths.journal_temporary, "restore journal temporary").await?; + + let Some(mut journal) = read_journal(&paths).await? else { + return reconcile_without_journal(&paths).await; + }; + if journal.instance_id != paths.instance_id { + return Err(storage_error(format!( + "restore journal instance '{}' does not match slot '{}'", + journal.instance_id, paths.instance_id + ))); + } + + match journal.state { + RestoreState::Committing => finish_commit(&paths).await, + RestoreState::Staged | RestoreState::Activated => { + journal.state = RestoreState::Aborting; + replace_journal(&paths, &journal).await?; + finish_abort(&paths).await + } + RestoreState::Aborting => finish_abort(&paths).await, + } +} + +async fn restore_paths(provider: &FileStorageProvider, instance_id: &str) -> Result { + let slot = provider.slot_for_id(instance_id)?; + let instances_dir = canonical_plain_path( + &provider.instances_dir, + RequiredPathType::Directory, + "instances directory", + ) + .await?; + let instance_dir = canonical_plain_path( + &slot.instance_dir, + RequiredPathType::Directory, + "slot directory", + ) + .await?; + if instance_dir.parent() != Some(instances_dir.as_path()) + || instance_dir.file_name() != Some(std::ffi::OsStr::new(instance_id)) + { + return Err(storage_error(format!( + "restore slot {} is not the direct '{}' child of instances directory {}", + instance_dir.display(), + instance_id, + instances_dir.display() + ))); + } + + Ok(RestorePaths { + instance_id: instance_id.to_string(), + rootfs: instance_dir.join("rootfs.ext4"), + copying: instance_dir.join(".rootfs.restore-copying"), + staged: instance_dir.join(".rootfs.restore-staged"), + backup: instance_dir.join(".rootfs.restore-backup"), + discard: instance_dir.join(".rootfs.restore-discard"), + journal: instance_dir.join(".rootfs.restore.json"), + journal_temporary: instance_dir.join(".rootfs.restore-journal.tmp"), + instance_dir, + }) +} + +async fn canonical_plain_path( + path: &Path, + required_type: RequiredPathType, + description: &str, +) -> Result { + let metadata = tokio::fs::symlink_metadata(path).await.map_err(|error| { + storage_error(format!("inspect {description} {}: {error}", path.display())) + })?; + if !required_type.matches(&metadata) || metadata.file_type().is_symlink() { + return Err(storage_error(format!( + "{description} {} is not a plain {}", + path.display(), + required_type.description() + ))); + } + tokio::fs::canonicalize(path).await.map_err(|error| { + storage_error(format!( + "canonicalize {description} {}: {error}", + path.display() + )) + }) +} + +async fn canonical_plain_file(path: &Path, description: &str) -> Result { + canonical_plain_path(path, RequiredPathType::File, description).await +} + +async fn require_plain_file(path: &Path, description: &str) -> Result<()> { + if plain_file_exists(path, description).await? { + Ok(()) + } else { + Err(storage_error(format!( + "{description} {} does not exist", + path.display() + ))) + } +} + +async fn plain_file_exists(path: &Path, description: &str) -> Result { + match tokio::fs::symlink_metadata(path).await { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => Ok(true), + Ok(_) => Err(storage_error(format!( + "{description} {} is not a plain file", + path.display() + ))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(storage_error(format!( + "inspect {description} {}: {error}", + path.display() + ))), + } +} + +async fn entry_exists(path: &Path) -> Result { + match tokio::fs::symlink_metadata(path).await { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(storage_error(format!( + "inspect restore artifact {}: {error}", + path.display() + ))), + } +} + +async fn ensure_no_transaction(paths: &RestorePaths) -> Result<()> { + for path in paths.transaction_artifacts() { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has unfinished restore artifact {}; reconcile it first", + paths.instance_id, + path.display() + ))); + } + } + Ok(()) +} + +async fn ensure_no_transient_files(paths: &RestorePaths) -> Result<()> { + for (path, description) in [ + (&paths.copying, "restore copying file"), + (&paths.journal_temporary, "restore journal temporary"), + ] { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has unfinished {description}; reconcile it first", + paths.instance_id + ))); + } + } + Ok(()) +} + +async fn copy_for_restore( + source: &Path, + destination: &Path, + unpublished: &mut UnpublishedFiles, +) -> Result<()> { + let mut source_options = tokio::fs::OpenOptions::new(); + source_options.read(true); + #[cfg(unix)] + source_options.custom_flags(libc::O_NOFOLLOW); + let mut source_file = source_options.open(source).await.map_err(|error| { + storage_error(format!("open restore source {}: {error}", source.display())) + })?; + if !source_file + .metadata() + .await + .map_err(|error| storage_error(format!("inspect restore source: {error}")))? + .is_file() + { + return Err(storage_error(format!( + "restore source {} is not a regular file", + source.display() + ))); + } + + let mut destination_options = tokio::fs::OpenOptions::new(); + destination_options.write(true).create_new(true); + #[cfg(unix)] + destination_options.custom_flags(libc::O_NOFOLLOW); + let mut destination_file = destination_options + .open(destination) + .await + .map_err(|error| { + storage_error(format!( + "create restore stage {}: {error}", + destination.display() + )) + })?; + unpublished.track(destination); + + let mut buffer = vec![0_u8; COPY_BUFFER_SIZE]; + let mut first_chunk = true; + loop { + let count = source_file + .read(&mut buffer) + .await + .map_err(|error| storage_error(format!("read restore source: {error}")))?; + if count == 0 { + break; + } + destination_file + .write_all(&buffer[..count]) + .await + .map_err(|error| storage_error(format!("write restore stage: {error}")))?; + if first_chunk { + first_chunk = false; + crate::failpoint::pause("storage-restore-copy-after-chunk").await; + } + } + destination_file + .sync_all() + .await + .map_err(|error| storage_error(format!("sync restore stage: {error}"))) +} + +async fn publish_new_journal( + paths: &RestorePaths, + journal: &RestoreJournal, + unpublished: &mut UnpublishedFiles, +) -> Result<()> { + let bytes = encode_journal(journal)?; + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let mut file = options + .open(&paths.journal_temporary) + .await + .map_err(|error| storage_error(format!("create restore journal: {error}")))?; + unpublished.track(&paths.journal_temporary); + file.write_all(&bytes) + .await + .map_err(|error| storage_error(format!("write restore journal: {error}")))?; + file.sync_all() + .await + .map_err(|error| storage_error(format!("sync restore journal: {error}")))?; + drop(file); + rename_new_plain_file(&paths.journal_temporary, &paths.journal).await?; + unpublished.untrack(&paths.journal_temporary); + // The journal now owns the staged rootfs, even if the directory sync fails. + unpublished.commit(); + Ok(()) +} + +async fn replace_journal(paths: &RestorePaths, journal: &RestoreJournal) -> Result<()> { + require_plain_file(&paths.journal, "restore journal").await?; + if entry_exists(&paths.journal_temporary).await? { + return Err(storage_error(format!( + "slot '{}' has an unfinished journal update; reconcile it first", + paths.instance_id + ))); + } + + let bytes = encode_journal(journal)?; + let mut cleanup = UnpublishedFiles::new(); + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let mut file = options + .open(&paths.journal_temporary) + .await + .map_err(|error| storage_error(format!("create restore journal update: {error}")))?; + cleanup.track(&paths.journal_temporary); + file.write_all(&bytes) + .await + .map_err(|error| storage_error(format!("write restore journal update: {error}")))?; + file.sync_all() + .await + .map_err(|error| storage_error(format!("sync restore journal update: {error}")))?; + drop(file); + tokio::fs::rename(&paths.journal_temporary, &paths.journal) + .await + .map_err(|error| storage_error(format!("replace restore journal: {error}")))?; + cleanup.untrack(&paths.journal_temporary); + sync_directory(&paths.instance_dir).await +} + +fn encode_journal(journal: &RestoreJournal) -> Result> { + serde_json::to_vec(journal) + .map_err(|error| storage_error(format!("encode restore journal: {error}"))) +} + +async fn read_journal(paths: &RestorePaths) -> Result> { + if !plain_file_exists(&paths.journal, "restore journal").await? { + return Ok(None); + } + let mut options = tokio::fs::OpenOptions::new(); + options.read(true); + #[cfg(unix)] + options.custom_flags(libc::O_NOFOLLOW); + let file = options + .open(&paths.journal) + .await + .map_err(|error| storage_error(format!("open restore journal: {error}")))?; + let mut bytes = Vec::new(); + file.take(MAX_JOURNAL_SIZE + 1) + .read_to_end(&mut bytes) + .await + .map_err(|error| storage_error(format!("read restore journal: {error}")))?; + if bytes.len() as u64 > MAX_JOURNAL_SIZE { + return Err(storage_error("restore journal exceeds the size limit")); + } + let journal: RestoreJournal = serde_json::from_slice(&bytes) + .map_err(|error| storage_error(format!("parse restore journal: {error}")))?; + if journal.version != JOURNAL_VERSION { + return Err(storage_error(format!( + "unsupported restore journal version {}", + journal.version + ))); + } + Ok(Some(journal)) +} + +async fn require_journal(paths: &RestorePaths) -> Result { + read_journal(paths).await?.ok_or_else(|| { + storage_error(format!( + "slot '{}' has no restore transaction", + paths.instance_id + )) + }) +} + +fn verify_transaction( + journal: &RestoreJournal, + transaction: &StorageRestoreTransaction, +) -> Result<()> { + if journal.instance_id != transaction.instance_id + || journal.transaction_id != transaction.transaction_id + { + return Err(storage_error(format!( + "restore transaction {} does not own slot '{}'", + transaction.transaction_id, transaction.instance_id + ))); + } + Ok(()) +} + +async fn inspect_layout(paths: &RestorePaths) -> Result<(bool, bool, bool, bool)> { + Ok(( + plain_file_exists(&paths.rootfs, "live rootfs").await?, + plain_file_exists(&paths.staged, "staged rootfs").await?, + plain_file_exists(&paths.backup, "retained rootfs").await?, + plain_file_exists(&paths.discard, "discarded rootfs").await?, + )) +} + +async fn ensure_activated_layout(paths: &RestorePaths) -> Result<()> { + if inspect_layout(paths).await? != (true, false, true, false) { + return Err(invalid_layout(paths, RestoreState::Activated)); + } + Ok(()) +} + +async fn ensure_finalized_layout(paths: &RestorePaths) -> Result<()> { + require_plain_file(&paths.rootfs, "live rootfs").await?; + for path in paths.transaction_artifacts() { + if entry_exists(path).await? { + return Err(storage_error(format!( + "slot '{}' has restore artifact {}; reconcile it first", + paths.instance_id, + path.display() + ))); + } + } + Ok(()) +} + +async fn finish_abort(paths: &RestorePaths) -> Result<()> { + for _ in 0..6 { + let layout = inspect_layout(paths).await?; + match layout { + (false, true, true, false) => { + rename_new_plain_file(&paths.backup, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-rollback-rootfs").await; + } + (true, true, false, false) => { + remove_plain_file(&paths.staged, "staged rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + (true, false, true, false) => { + rename_new_plain_file(&paths.rootfs, &paths.discard).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-discard").await; + } + (false, false, true, true) => { + rename_new_plain_file(&paths.backup, &paths.rootfs).await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-rollback-rootfs").await; + } + (true, false, false, true) => { + remove_plain_file(&paths.discard, "discarded rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + (true, false, false, false) => { + remove_plain_file(&paths.journal, "restore journal").await?; + sync_directory(&paths.instance_dir).await?; + return Ok(()); + } + _ => return Err(invalid_layout(paths, RestoreState::Aborting)), + } + } + Err(storage_error(format!( + "restore abort for '{}' did not converge", + paths.instance_id + ))) +} + +async fn finish_commit(paths: &RestorePaths) -> Result<()> { + let (live, staged, _backup, discard) = inspect_layout(paths).await?; + if !live || staged || discard { + return Err(invalid_layout(paths, RestoreState::Committing)); + } + remove_plain_file_if_present(&paths.backup, "retained rootfs").await?; + sync_directory(&paths.instance_dir).await?; + crate::failpoint::pause("storage-restore-after-backup-release").await; + remove_plain_file(&paths.journal, "restore journal").await?; + sync_directory(&paths.instance_dir).await +} + +async fn reconcile_without_journal(paths: &RestorePaths) -> Result<()> { + let (live, staged, backup, discard) = inspect_layout(paths).await?; + if backup || discard || !live { + return Err(storage_error(format!( + "slot '{}' has ambiguous restore artifacts without a journal", + paths.instance_id + ))); + } + if staged { + remove_plain_file(&paths.staged, "unpublished staged rootfs").await?; + sync_directory(&paths.instance_dir).await?; + } + Ok(()) +} + +async fn rename_new_plain_file(source: &Path, target: &Path) -> Result<()> { + require_plain_file(source, "restore rename source").await?; + if entry_exists(target).await? { + return Err(storage_error(format!( + "restore rename target {} already exists", + target.display() + ))); + } + tokio::fs::rename(source, target).await.map_err(|error| { + storage_error(format!( + "rename restore file {} to {}: {error}", + source.display(), + target.display() + )) + }) +} + +async fn remove_plain_file(path: &Path, description: &str) -> Result<()> { + require_plain_file(path, description).await?; + tokio::fs::remove_file(path) + .await + .map_err(|error| storage_error(format!("remove {description} {}: {error}", path.display()))) +} + +async fn remove_plain_file_if_present(path: &Path, description: &str) -> Result<()> { + if plain_file_exists(path, description).await? { + remove_plain_file(path, description).await?; + } + Ok(()) +} + +async fn sync_directory(path: &Path) -> Result<()> { + tokio::fs::File::open(path) + .await + .map_err(|error| { + storage_error(format!( + "open restore directory {}: {error}", + path.display() + )) + })? + .sync_all() + .await + .map_err(|error| { + storage_error(format!( + "sync restore directory {}: {error}", + path.display() + )) + }) +} + +fn invalid_layout(paths: &RestorePaths, state: RestoreState) -> BlazeError { + storage_error(format!( + "slot '{}' has an invalid {:?} restore layout", + paths.instance_id, state + )) +} + +fn storage_error(message: impl Into) -> BlazeError { + BlazeError::StorageError { + msg: message.into(), + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use async_trait::async_trait; + use blaze_core::storage::{ + AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + }; + + use super::*; + + struct UnsupportedStorage; + + #[async_trait] + impl StorageProvider for UnsupportedStorage { + async fn probe(&self) -> Result { + Ok(true) + } + + async fn acquire( + &self, + _opts: &AcquireOpts, + ) -> std::result::Result { + Err(StorageAcquireError::clean(storage_error( + "acquire unavailable", + ))) + } + + async fn release(&self, _slot: StorageSlot) -> Result<()> { + Ok(()) + } + + async fn reconstruct(&self, _instance_id: &str) -> Result { + Err(storage_error("reconstruct unavailable")) + } + + async fn flush_dirty(&self, _slot: &StorageSlot) -> Result<()> { + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + PoolStatus::default() + } + + async fn drain_pool(&self) -> Result { + Ok(0) + } + } + + async fn fixture( + instance_id: &str, + ) -> (tempfile::TempDir, FileStorageProvider, StorageSlot, PathBuf) { + let temp = tempfile::tempdir().expect("temporary storage"); + let instances = temp.path().join("instances"); + let checkpoints = temp.path().join("checkpoints"); + tokio::fs::create_dir(&instances) + .await + .expect("instances directory"); + tokio::fs::create_dir(&checkpoints) + .await + .expect("checkpoints directory"); + let provider = FileStorageProvider::new(instances); + let slot = provider + .acquire(&AcquireOpts { + instance_id: instance_id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("storage slot"); + tokio::fs::write(&slot.rootfs_path, b"live-rootfs") + .await + .expect("live rootfs"); + let source = checkpoints.join("rootfs.snap"); + tokio::fs::write(&source, b"checkpoint-rootfs") + .await + .expect("checkpoint rootfs"); + (temp, provider, slot, source) + } + + async fn rootfs(path: &Path) -> Vec { + tokio::fs::read(path).await.expect("read rootfs") + } + + #[tokio::test] + async fn restore_contract_is_opt_in_and_fail_closed() { + let provider = UnsupportedStorage; + let slot = StorageSlot { + id: "unsupported".to_string(), + rootfs_path: PathBuf::from("rootfs"), + mem_path: PathBuf::from("memory"), + mem_diff_path: PathBuf::from("memory-diff"), + rootfs_diff_path: PathBuf::from("rootfs-diff"), + instance_dir: PathBuf::from("instance"), + }; + let transaction = StorageRestoreTransaction { + instance_id: slot.id.clone(), + transaction_id: Uuid::new_v4(), + }; + + assert!(!provider.supports_checkpoint_restore()); + assert!( + provider + .stage_checkpoint_restore(&slot, Path::new("checkpoint")) + .await + .is_err() + ); + assert!( + provider + .activate_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .commit_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .abort_checkpoint_restore(&transaction) + .await + .is_err() + ); + assert!( + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .is_err() + ); + } + + #[tokio::test] + async fn stage_keeps_the_live_rootfs_running_image_unchanged() { + let (_temp, provider, slot, source) = fixture("stage-independent").await; + + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage restore"); + + assert!(provider.supports_checkpoint_restore()); + assert_eq!(transaction.instance_id, slot.id); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("a second transaction must fail closed"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort staged restore"); + } + + #[tokio::test] + async fn activated_restore_can_be_aborted_to_the_predecessor() { + let (_temp, provider, slot, source) = fixture("activate-abort").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&provider, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[cfg(unix)] + #[tokio::test] + async fn activation_retains_the_original_rootfs_inode_until_finalization() { + use std::os::unix::fs::MetadataExt; + + let (_temp, provider, slot, source) = fixture("retain-inode").await; + let original_inode = tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("live metadata") + .ino(); + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + assert_eq!( + tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("staged live metadata") + .ino(), + original_inode + ); + + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + assert_eq!( + tokio::fs::metadata(&paths.backup) + .await + .expect("backup metadata") + .ino(), + original_inode + ); + assert_ne!( + tokio::fs::metadata(&paths.rootfs) + .await + .expect("selected metadata") + .ino(), + original_inode + ); + + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + assert_eq!( + tokio::fs::metadata(&slot.rootfs_path) + .await + .expect("restored metadata") + .ino(), + original_inode + ); + } + + #[tokio::test] + async fn committed_restore_releases_the_predecessor() { + let (_temp, provider, slot, source) = fixture("activate-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + provider + .commit_checkpoint_restore(&transaction) + .await + .expect("commit"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + ensure_finalized_layout(&restore_paths(&provider, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn stale_transaction_handle_cannot_select_a_rootfs() { + let (_temp, provider, slot, source) = fixture("stale-handle").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let stale = StorageRestoreTransaction { + instance_id: transaction.instance_id.clone(), + transaction_id: Uuid::new_v4(), + }; + + provider + .activate_checkpoint_restore(&stale) + .await + .expect_err("stale transaction must be rejected"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + } + + #[tokio::test] + async fn staging_rederives_provider_paths_from_the_slot_id() { + let (temp, provider, slot, source) = fixture("canonical-slot").await; + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external") + .await + .expect("external rootfs"); + let mut forged = slot.clone(); + forged.rootfs_path = external.clone(); + forged.instance_dir = temp.path().to_path_buf(); + + let transaction = provider + .stage_checkpoint_restore(&forged, &source) + .await + .expect("stage through canonical slot"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + assert_eq!(rootfs(&external).await, b"external"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_linked_sources_and_slot_paths() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-paths").await; + let linked_source = temp.path().join("linked-source"); + symlink(&source, &linked_source).expect("source link"); + provider + .stage_checkpoint_restore(&slot, &linked_source) + .await + .expect_err("linked source must be rejected"); + + tokio::fs::remove_file(&slot.rootfs_path) + .await + .expect("remove live rootfs"); + let external = temp.path().join("external-rootfs"); + tokio::fs::write(&external, b"external") + .await + .expect("external rootfs"); + symlink(&external, &slot.rootfs_path).expect("rootfs link"); + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked live rootfs must be rejected"); + assert_eq!(rootfs(&external).await, b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_a_linked_slot_directory() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-slot").await; + tokio::fs::remove_dir_all(&slot.instance_dir) + .await + .expect("remove slot"); + let external = temp.path().join("external-slot"); + tokio::fs::create_dir(&external) + .await + .expect("external slot"); + tokio::fs::write(external.join("rootfs.ext4"), b"external") + .await + .expect("external rootfs"); + symlink(&external, &slot.instance_dir).expect("slot link"); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked slot directory must be rejected"); + + assert_eq!(rootfs(&external.join("rootfs.ext4")).await, b"external"); + } + + #[cfg(unix)] + #[tokio::test] + async fn staging_rejects_linked_transaction_artifacts() { + use std::os::unix::fs::symlink; + + let (temp, provider, slot, source) = fixture("linked-artifact").await; + let external = temp.path().join("external"); + tokio::fs::write(&external, b"external") + .await + .expect("external"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + symlink(&external, &paths.staged).expect("stage link"); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("linked transaction artifact must fail closed"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert_eq!(rootfs(&external).await, b"external"); + } + + #[tokio::test] + async fn staging_rejects_instance_id_path_components() { + let (_temp, provider, mut slot, source) = fixture("valid-id").await; + slot.id = "../escape".to_string(); + + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect_err("path component must be rejected"); + } + + #[tokio::test] + async fn restart_aborts_a_staged_restore() { + let (_temp, provider, slot, source) = fixture("restart-staged").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn restart_aborts_an_activated_restore() { + let (_temp, provider, slot, source) = fixture("restart-activated").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[tokio::test] + async fn restart_finishes_a_durable_commit_intent() { + let (_temp, provider, slot, source) = fixture("restart-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + let mut journal = require_journal(&paths).await.expect("journal"); + journal.state = RestoreState::Committing; + replace_journal(&paths, &journal) + .await + .expect("commit intent"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &slot.id).await.unwrap()) + .await + .unwrap(); + } + + #[tokio::test] + async fn restart_cleans_a_partial_copy_without_touching_the_live_rootfs() { + let (_temp, provider, slot, _source) = fixture("restart-copying").await; + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + tokio::fs::write(&paths.copying, b"partial") + .await + .expect("partial copy"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert!(!paths.copying.exists()); + } + + #[tokio::test] + async fn restart_recovers_after_retaining_the_predecessor() { + let (_temp, provider, slot, source) = fixture("restart-after-backup").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + rename_new_plain_file(&paths.rootfs, &paths.backup) + .await + .expect("retain predecessor"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + assert!(!paths.staged.exists()); + assert!(!paths.backup.exists()); + } + + #[tokio::test] + async fn restart_recovers_after_switching_before_journal_update() { + let (_temp, provider, slot, source) = fixture("restart-after-switch").await; + provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + rename_new_plain_file(&paths.rootfs, &paths.backup) + .await + .expect("retain predecessor"); + rename_new_plain_file(&paths.staged, &paths.rootfs) + .await + .expect("switch rootfs"); + + let restarted = FileStorageProvider::new(provider.instances_dir.clone()); + restarted + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[tokio::test] + async fn corrupt_journal_preserves_both_rootfs_versions() { + let (_temp, provider, slot, source) = fixture("corrupt-journal").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + tokio::fs::write(&paths.journal, b"not-json") + .await + .expect("corrupt journal"); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect_err("corrupt journal must fail closed"); + + assert_eq!(rootfs(&paths.rootfs).await, b"checkpoint-rootfs"); + assert_eq!(rootfs(&paths.backup).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + async fn cancel_at( + provider: FileStorageProvider, + slot: StorageSlot, + source: PathBuf, + failpoint: &'static str, + ) { + let hook = crate::failpoint::TestFailpoint::new(&[failpoint]); + let operation_hook = hook.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(provider.stage_checkpoint_restore(&slot, &source)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("operation must be cancelled") + .is_cancelled() + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_partial_copy_is_removed() { + let (_temp, provider, slot, source) = fixture("cancel-copy").await; + tokio::fs::write(&source, vec![42_u8; COPY_BUFFER_SIZE * 4]) + .await + .expect("large checkpoint"); + let instances = provider.instances_dir.clone(); + let id = slot.id.clone(); + let rootfs_path = slot.rootfs_path.clone(); + + cancel_at(provider, slot, source, "storage-restore-copy-after-chunk").await; + + let restarted = FileStorageProvider::new(instances); + restarted + .reconcile_checkpoint_restore(&id) + .await + .expect("reconcile cancellation"); + assert_eq!(rootfs(&rootfs_path).await, b"live-rootfs"); + ensure_finalized_layout(&restore_paths(&restarted, &id).await.unwrap()) + .await + .unwrap(); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn failed_second_rename_immediately_restores_the_predecessor() { + let (_temp, provider, slot, source) = fixture("failed-switch").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-switch"]); + + hook.run(provider.activate_checkpoint_restore(&transaction)) + .await + .expect_err("selecting the staged rootfs must fail"); + + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + provider + .abort_checkpoint_restore(&transaction) + .await + .expect("abort retained stage"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn failed_switch_compensation_remains_reconcilable() { + let (_temp, provider, slot, source) = fixture("failed-switch-rollback").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&[ + "storage-restore-switch", + "storage-restore-switch-rollback", + ]); + + hook.run(provider.activate_checkpoint_restore(&transaction)) + .await + .expect_err("selection and immediate compensation must fail"); + + let paths = restore_paths(&provider, &slot.id).await.expect("paths"); + assert!(!paths.rootfs.exists()); + assert_eq!(rootfs(&paths.backup).await, b"live-rootfs"); + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile retained predecessor"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_activation_after_backup_is_reconciled() { + let (_temp, provider, slot, source) = fixture("cancel-activation").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-backup"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.activate_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("activation must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_commit_intent_is_completed_on_restart() { + let (_temp, provider, slot, source) = fixture("cancel-commit").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-commit-intent"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.commit_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("commit must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"checkpoint-rootfs"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_abort_is_completed_on_restart() { + let (_temp, provider, slot, source) = fixture("cancel-abort").await; + let transaction = provider + .stage_checkpoint_restore(&slot, &source) + .await + .expect("stage"); + provider + .activate_checkpoint_restore(&transaction) + .await + .expect("activate"); + let hook = crate::failpoint::TestFailpoint::new(&["storage-restore-after-discard"]); + let operation_hook = hook.clone(); + let operation_provider = FileStorageProvider::new(provider.instances_dir.clone()); + let operation_transaction = transaction.clone(); + let operation = tokio::spawn(async move { + operation_hook + .run(operation_provider.abort_checkpoint_restore(&operation_transaction)) + .await + }); + hook.wait_until_paused().await; + operation.abort(); + assert!( + operation + .await + .expect_err("abort must be cancelled") + .is_cancelled() + ); + + provider + .reconcile_checkpoint_restore(&slot.id) + .await + .expect("reconcile"); + assert_eq!(rootfs(&slot.rootfs_path).await, b"live-rootfs"); + } +} From 1f3b685b08c055b7f63712acf22161218af3085e Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 11:39:30 +0800 Subject: [PATCH 32/40] feat(blaze): restore sandbox checkpoints Blaze could capture and list full checkpoints, but the daemon had no route that selected one as the live runtime state. Callers could inspect history but could not replace a running sandbox from that history. Add opt-in storage and backend restore contracts, Firecracker and mock adapters, a verified checkpoint target, and the manager-owned rollback API. This enables callers to replace a running sandbox from a selected checkpoint. The file provider stages an independent rootfs while the old backend runs, retains the predecessor through activation, and commits only after the new owner is ready and HEAD has changed. Firecracker requires the exact captured binary version before starting a replacement process. Failures before backend shutdown restore the original running state. Later failures retain the resources that actually exist under RecoveryRequired, including partial owners and uncertain HEAD outcomes, so destroy can finish cleanup without losing ownership. The contracts, adapters, manager, API, tests, and documentation stay together because they form the first production restore path; none is independently usable without the others. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 19 + src/blaze/README_zh.md | 15 + src/blaze/crates/blaze-core/src/backend.rs | 36 ++ src/blaze/crates/blazed/src/api.rs | 518 +++++++++++++++++- .../crates/blazed/src/checkpoint_store.rs | 32 ++ src/blaze/crates/blazed/src/sandbox.rs | 2 + .../crates/blazed/src/sandbox/manager.rs | 19 +- .../crates/blazed/src/sandbox/restore.rs | 497 +++++++++++++++++ src/blaze/crates/blazed/src/spawner.rs | 188 ++++++- .../crates/blazed/src/spawner/firecracker.rs | 467 +++++++++++++++- 10 files changed, 1756 insertions(+), 37 deletions(-) create mode 100644 src/blaze/crates/blazed/src/sandbox/restore.rs diff --git a/src/blaze/README.md b/src/blaze/README.md index cbd66a0729..6baed6c9ad 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -140,6 +140,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/sandboxes/{id}/write` | Replace a guest file | | POST | `/v1/sandboxes/{id}/checkpoint` | Capture a full checkpoint when the backend and storage provider support it | | GET | `/v1/sandboxes/{id}/checkpoints` | List committed checkpoints and HEAD reachability | +| POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | Replace a running sandbox from a verified checkpoint | | GET | `/v1/instances` | Alias for listing sandboxes | | POST | `/v1/instances` | Alias for creating a sandbox | | GET | `/v1/instances/{id}` | Alias for sandbox details | @@ -150,6 +151,7 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances/{id}/write` | Compatible guest file write action | | POST | `/v1/instances/{id}/checkpoint` | Compatible full-checkpoint action | | GET | `/v1/instances/{id}/checkpoints` | Compatible checkpoint-list action | +| POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | Compatible checkpoint-restore action | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | @@ -210,6 +212,23 @@ remain available for explicit cleanup. Listing uses the same per-sandbox operation lock as capture, guest operations, and destroy. Destroy removes transaction scratch but preserves committed checkpoint history. +Checkpoint restore is available only when the current storage provider and the +checkpoint's backend implement restore, and the current backend version exactly +matches the version recorded at capture. The daemon verifies the selected +checkpoint, its parent chain, and all artifact hashes before changing runtime +state. + +The file provider stages a separate rootfs copy while the current backend is +still running. After the old backend stops, the daemon selects that copy, +starts and owns the replacement backend, moves HEAD to the selected checkpoint, +and only then releases the previous rootfs. A failure before backend shutdown +keeps the original runtime running. A failure after shutdown retains the +resources that actually exist and marks the sandbox `RecoveryRequired`, so a +later destroy can finish cleanup without losing process ownership. + +`last_checkpoint` continues to mean the most recent completed capture. Restore +moves catalog HEAD but does not rewrite capture history. + Runtime reset remains reserved and returns `501` without changing runtime or persisted state. diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index bf34d8d52f..d8b5662b90 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -136,6 +136,7 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/sandboxes/{id}/write` | 替换 guest 文件 | | POST | `/v1/sandboxes/{id}/checkpoint` | 后端和存储 provider 支持时捕获完整 checkpoint | | GET | `/v1/sandboxes/{id}/checkpoints` | 列出已提交的 checkpoint 及其 HEAD 可达性 | +| POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | 用经过校验的 checkpoint 替换正在运行的 sandbox | | GET | `/v1/instances` | 列出 sandbox 的兼容入口 | | POST | `/v1/instances` | 创建 sandbox 的兼容入口 | | GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | @@ -146,6 +147,7 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances/{id}/write` | Guest 文件写入兼容入口 | | POST | `/v1/instances/{id}/checkpoint` | 捕获完整 checkpoint 的兼容入口 | | GET | `/v1/instances/{id}/checkpoints` | 列出 checkpoint 的兼容入口 | +| POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | 恢复 checkpoint 的兼容入口 | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | @@ -197,6 +199,19 @@ stage。如果发布或 HEAD 的结果无法确定，或者后端无法恢复， 保留，供后续显式清理。列出 checkpoint 与捕获、guest 操作及销毁共用同一个 sandbox 操作锁。销毁会删除事务临时文件，但保留已经提交的 checkpoint 历史。 +只有当前存储 provider 和 checkpoint 对应的后端都实现恢复，并且当前后端版本 +与捕获时记录的版本完全一致，daemon 才会开始恢复。修改 runtime 之前，daemon +会先校验所选 checkpoint、完整父链和全部 artifact hash。 + +file provider 会在旧后端仍然运行时准备一份独立的 rootfs。旧后端停止后， +daemon 才选择这份 rootfs，启动并持有新的后端，随后把 HEAD 指向所选 +checkpoint，最后释放旧 rootfs。旧后端停止前发生失败时，原 runtime 会继续 +运行；停止后发生任何无法确认的失败时，daemon 会保留实际存在的资源，并把 +sandbox 标记为 `RecoveryRequired`，后续 destroy 仍能找到并清理这些资源。 + +`last_checkpoint` 始终表示最近一次成功捕获。恢复只移动 catalog HEAD，不会 +改写捕获历史。 + runtime reset 仍是预留接口，会返回 `501`，且不会修改 runtime 或持久化状态。 ### Guest 操作 diff --git a/src/blaze/crates/blaze-core/src/backend.rs b/src/blaze/crates/blaze-core/src/backend.rs index f869f6d5ce..30ebeeda5b 100644 --- a/src/blaze/crates/blaze-core/src/backend.rs +++ b/src/blaze/crates/blaze-core/src/backend.rs @@ -101,6 +101,42 @@ pub struct SpawnRequest { pub vm: Option, } +/// Backend identity and snapshot semantics accepted by a restore adapter. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RestoreCapability { + /// Concrete backend implementation that can consume the checkpoint. + pub backend: BackendKind, + /// Exact backend version required by versioned snapshot formats. + pub version: Option, + /// Snapshot flavor accepted by the adapter. + pub snapshot_kind: SnapshotKind, +} + +/// Complete input for restoring an owned backend instance. +#[derive(Debug, Clone)] +pub struct RestoreRequest { + /// Stable sandbox identifier. + pub instance_id: Uuid, + /// Provider-owned runtime directory that preserves backend resource paths. + pub run_dir: PathBuf, + /// Backend executable selected from the current daemon configuration. + pub binary_path: PathBuf, + /// Storage resources reconstructed for this sandbox. + pub storage: StorageSlot, + /// VM-state artifact from a committed checkpoint. + pub snapshot_path: PathBuf, + /// Guest-memory artifact from the same checkpoint. + pub mem_path: PathBuf, + /// Backend identity frozen into the checkpoint metadata. + pub checkpoint_backend: BackendKind, + /// Backend version frozen into the checkpoint metadata. + pub expected_version: Option, + /// Snapshot flavor frozen into the checkpoint metadata. + pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable run-directory guest transport. + pub expose_guest_socket: bool, +} + /// Snapshot flavor requested from a backend. /// /// The file provider currently requires self-contained artifacts, so only diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index a7bb54fe02..f5a9096c0b 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -31,7 +31,7 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::guest::MAX_GUEST_FILE_BYTES; use crate::request_body; -use crate::sandbox::CreateSandbox; +use crate::sandbox::{CreateSandbox, RestoreSandbox, RestoreSandboxResult}; use crate::state::ServerState; const MAX_EXEC_TIMEOUT_SECS: u32 = 20; @@ -122,6 +122,10 @@ async fn dispatch( | ("POST", ["v1", "sandboxes", id, "checkpoint"]) => checkpoint(state, id).await, ("GET", ["v1", "instances", id, "checkpoints"]) | ("GET", ["v1", "sandboxes", id, "checkpoints"]) => list_checkpoints(state, id).await, + ("POST", ["v1", "instances", id, "rollback", checkpoint_id]) + | ("POST", ["v1", "sandboxes", id, "rollback", checkpoint_id]) => { + rollback(state, id, checkpoint_id).await + } ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, ("DELETE", ["v1", "instances", id]) | ("DELETE", ["v1", "sandboxes", id]) @@ -312,6 +316,39 @@ async fn list_checkpoints(state: &Arc, id: &str) -> Result, + id: &str, + checkpoint_id: &str, +) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(instance.backend.as_str()) + .cloned() + .unwrap_or_default(); + let restored: RestoreSandboxResult = state + .manager + .restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint_id.to_string(), + binary_path, + }, + ) + .await?; + json_ok(&json!({ + "instance_id": restored.instance.id, + "checkpoint_id": restored.checkpoint_id, + "restored": true, + "state": restored.instance.state, + })) +} + async fn reset_instance(state: &Arc, id: &str) -> Result>> { let uuid = parse_uuid(id)?; let _operation = state @@ -1491,6 +1528,30 @@ mod tests { } } + struct CaptureOnlyMockSpawner; + + #[async_trait] + impl BackendSpawner for CaptureOnlyMockSpawner { + async fn spawn( + &self, + request: SpawnRequest, + ) -> std::result::Result { + MockSpawner.spawn(request).await + } + + async fn probe(&self, _binary_path: &Path) -> blaze_core::Result { + Ok(true) + } + + async fn cleanup_orphan( + &self, + instance_id: Uuid, + run_dir: &Path, + ) -> blaze_core::Result<()> { + MockSpawner.cleanup_orphan(instance_id, run_dir).await + } + } + struct StalledGuestOwner { instance_id: Uuid, socket: PathBuf, @@ -2033,6 +2094,461 @@ mod tests { assert!(state.manager.backend_owner(uuid).is_some()); } + #[tokio::test] + async fn rollback_replaces_runtime_state_without_rewriting_capture_history() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = state.storage.reconstruct(id).await.expect("storage slot"); + + tokio::fs::write(&slot.rootfs_path, b"first-rootfs") + .await + .expect("first rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"first-memory") + .await + .expect("first guest state"); + let (_, first) = dispatched_json( + &state, + Method::POST, + &format!("/v1/instances/{id}/checkpoint"), + Vec::new(), + ) + .await; + let first_id = first["id"].as_str().expect("first checkpoint"); + + tokio::fs::write(&slot.rootfs_path, b"second-rootfs") + .await + .expect("second rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"second-memory") + .await + .expect("second guest state"); + let (_, second) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/checkpoint"), + Vec::new(), + ) + .await; + let second_id = second["id"].as_str().expect("second checkpoint"); + + tokio::fs::write(&slot.rootfs_path, b"third-rootfs") + .await + .expect("third rootfs"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"third-memory") + .await + .expect("third guest state"); + + let (status, restored) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/rollback/{first_id}"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(restored["instance_id"], id); + assert_eq!(restored["checkpoint_id"], first_id); + assert_eq!(restored["restored"], true); + assert_eq!(restored["state"], "running"); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("restored rootfs"), + b"first-rootfs" + ); + assert_eq!( + state + .manager + .read_file(uuid, "/tmp/value".to_string()) + .await + .expect("restored guest state"), + b"first-memory" + ); + + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + assert_eq!(lifecycle.last_checkpoint.as_deref(), Some(second_id)); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint list") + .iter() + .find(|checkpoint| checkpoint.is_head) + .map(|checkpoint| checkpoint.id.as_str()), + Some(first_id) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + for name in [ + ".rootfs.restore-copying", + ".rootfs.restore-staged", + ".rootfs.restore-backup", + ".rootfs.restore-discard", + ".rootfs.restore.json", + ".rootfs.restore-journal.tmp", + ] { + assert!(!slot.instance_dir.join(name).exists(), "{name} remains"); + } + } + + #[tokio::test] + async fn rollback_rejects_an_unavailable_adapter_before_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(CaptureOnlyMockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + + let error = state + .manager + .restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("restore must require an adapter"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn restore_stage_failure_keeps_the_current_runtime_running() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-storage-stage"]); + + hook.run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("stage failure"); + + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_backend_stop_retains_the_current_owner_and_rootfs() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let owner = state.manager.backend_owner(uuid).expect("backend owner"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-backend-stop"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("backend stop outcome must require recovery"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("unchanged rootfs"), + b"current-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::RestoreStorageStaged) + ); + for name in [ + ".rootfs.restore-staged", + ".rootfs.restore-backup", + ".rootfs.restore.json", + ] { + assert!(!slot.instance_dir.join(name).exists(), "{name} remains"); + } + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_head_update_retains_the_replacement_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"later-checkpoint-rootfs") + .await + .expect("later checkpoint rootfs"); + let latest = state + .manager + .checkpoint(uuid) + .await + .expect("later checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let hook = crate::failpoint::TestFailpoint::new(&["checkpoint-store-head-after-rename"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id.clone(), + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("HEAD update must be reported"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("selected rootfs"), + b"checkpoint-rootfs" + ); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::RestoreBackendStarted) + ); + assert_eq!( + lifecycle.last_checkpoint.as_deref(), + Some(latest.id.as_str()) + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("observable checkpoint catalog") + .iter() + .find(|item| item.is_head) + .map(|item| item.id.as_str()), + Some(checkpoint.id.as_str()) + ); + + assert!(state.manager.destroy(uuid).await.expect("destroy")); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn final_state_failure_keeps_the_committed_restore_journal() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let slot = write_checkpoint_fixture(&state, id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + tokio::fs::write(&slot.rootfs_path, b"current-rootfs") + .await + .expect("current rootfs"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-final-state"]); + + let error = hook + .run(state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id.clone(), + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("final state failure"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert_eq!( + tokio::fs::read(&slot.rootfs_path) + .await + .expect("committed rootfs"), + b"checkpoint-rootfs" + ); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert_eq!( + lifecycle + .operation + .as_ref() + .map(|operation| (operation.checkpoint_id.as_deref(), operation.phase)), + Some(( + Some(checkpoint.id.as_str()), + Some(OperationPhase::RestoreStorageCommitted) + )) + ); + assert_eq!( + state + .manager + .list_checkpoints(uuid) + .await + .expect("checkpoint list") + .iter() + .find(|item| item.is_head) + .map(|item| item.id.as_str()), + Some(checkpoint.id.as_str()) + ); + assert!(state.manager.backend_owner(uuid).is_some()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn cancelled_restore_after_head_is_destroyable() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id").to_string(); + let uuid = Uuid::parse_str(&id).expect("uuid"); + write_checkpoint_fixture(&state, &id).await; + let checkpoint = state.manager.checkpoint(uuid).await.expect("checkpoint"); + let hook = crate::failpoint::TestFailpoint::new(&["restore-after-head"]); + let restore_state = state.clone(); + let restore_hook = hook.clone(); + let restore = tokio::spawn(async move { + restore_hook + .run(restore_state.manager.restore( + uuid, + RestoreSandbox { + checkpoint_id: checkpoint.id, + binary_path: PathBuf::new(), + }, + )) + .await + }); + hook.wait_until_paused().await; + + let persisted = + SandboxInstance::load(&state.state_dir, uuid).expect("persisted restore journal"); + assert_eq!(persisted.state, SandboxState::Restoring); + assert_eq!( + persisted.operation.and_then(|operation| operation.phase), + Some(OperationPhase::RestoreHeadUpdated) + ); + assert_eq!(persisted.backend_ownership, BackendOwnership::Running); + assert!(state.manager.backend_owner(uuid).is_some()); + + restore.abort(); + assert!(restore.await.expect_err("cancelled restore").is_cancelled()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + assert_eq!( + state.manager.get(uuid).expect("destroyed").state, + SandboxState::Destroyed + ); + assert!( + !state + .config + .lock() + .expect("config") + .storage + .instances_dir + .join(id) + .exists() + ); + } + #[cfg(feature = "test-failpoints")] #[tokio::test] async fn checkpoint_snapshot_failure_resumes_and_clears_the_journal() { diff --git a/src/blaze/crates/blazed/src/checkpoint_store.rs b/src/blaze/crates/blazed/src/checkpoint_store.rs index fb0ec265c2..3fbfaf82e2 100644 --- a/src/blaze/crates/blazed/src/checkpoint_store.rs +++ b/src/blaze/crates/blazed/src/checkpoint_store.rs @@ -58,6 +58,19 @@ pub enum CheckpointStoreError { /// Convenient result type for checkpoint catalog operations. pub type Result = std::result::Result; +/// Verified checkpoint metadata and provider-safe artifact paths. +#[derive(Debug)] +pub struct VerifiedCheckpoint { + /// Validated checkpoint manifest. + pub metadata: CheckpointMetadata, + /// Full backend-state snapshot. + pub snapshot_path: PathBuf, + /// Full guest-memory snapshot. + pub memory_path: PathBuf, + /// Self-contained root filesystem snapshot. + pub rootfs_path: PathBuf, +} + /// Temporary checkpoint directory populated before atomic publication. #[derive(Debug)] pub struct CheckpointStage { @@ -231,6 +244,25 @@ impl CheckpointStore { Ok(metadata) } + /// Verify a restore target, its complete ancestry, and its artifact paths. + pub fn verify_restore_target( + &self, + sandbox_id: Uuid, + checkpoint_id: &str, + ) -> Result { + let metadata = self.verify(sandbox_id, checkpoint_id)?; + if let Some(parent) = metadata.parent.as_deref() { + self.validated_chain_from(sandbox_id, parent)?; + } + let directory = self.committed_dir(sandbox_id, checkpoint_id)?; + Ok(VerifiedCheckpoint { + snapshot_path: require_contained_file(&directory, "vmstate.snap")?, + memory_path: require_contained_file(&directory, "memory.snap")?, + rootfs_path: require_contained_file(&directory, "rootfs.snap")?, + metadata, + }) + } + /// List committed checkpoints and mark the lineage reachable from HEAD. pub fn list(&self, sandbox_id: Uuid) -> Result> { let Some(_) = self.optional_sandbox_dir(sandbox_id)? else { diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs index 93504747dd..42ad6248c2 100644 --- a/src/blaze/crates/blazed/src/sandbox.rs +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -3,5 +3,7 @@ mod checkpoint; mod manager; +mod restore; pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; +pub use restore::{RestoreSandbox, RestoreSandboxResult}; diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 72ec540dc4..7891a818d8 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -22,7 +22,7 @@ use crate::checkpoint_store::CheckpointStore; use crate::error::{BlazeDaemonError, Result}; use crate::guest::{GuestClient, GuestExecResult, MAX_GUEST_FILE_BYTES}; use crate::metrics::Metrics; -use crate::spawner::{DynBackendInstance, SpawnerRegistry}; +use crate::spawner::{DynBackendInstance, DynSpawner, SpawnerRegistry}; const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); const GUEST_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); @@ -196,6 +196,21 @@ impl SandboxManager { } } + pub(super) fn spawner(&self, backend: BackendKind) -> Option { + self.spawners.get(backend) + } + + pub(super) fn runtime_dir(&self, id: Uuid) -> PathBuf { + self.state_dir.join(id.to_string()) + } + + pub(super) fn remove_backend_owner(&self, id: Uuid) -> Option { + match self.backend_instances.lock() { + Ok(mut instances) => instances.remove(&id), + Err(poisoned) => poisoned.into_inner().remove(&id), + } + } + #[cfg(test)] pub fn insert_backend_owner(&self, id: Uuid, backend: DynBackendInstance) -> Result<()> { self.backend_instances @@ -1247,7 +1262,7 @@ impl SandboxManager { } } - fn retain_backend(&self, id: Uuid, backend: DynBackendInstance) -> Option { + pub(super) fn retain_backend(&self, id: Uuid, backend: DynBackendInstance) -> Option { match self.backend_instances.lock() { Ok(mut instances) => { instances.insert(id, backend); diff --git a/src/blaze/crates/blazed/src/sandbox/restore.rs b/src/blaze/crates/blazed/src/sandbox/restore.rs new file mode 100644 index 0000000000..63c4044e37 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/restore.rs @@ -0,0 +1,497 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Recoverable replacement of a running sandbox from a committed checkpoint. + +use std::path::PathBuf; + +use blaze_core::backend::{RestoreRequest, SnapshotKind}; +use blaze_core::checkpoint::validate_checkpoint_id; +use blaze_core::lifecycle::{BackendOwnership, OperationPhase, SandboxInstance, SandboxState}; +use blaze_core::storage::StorageRestoreTransaction; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +/// Inputs resolved from the current daemon configuration. +#[derive(Debug, Clone)] +pub struct RestoreSandbox { + /// Committed checkpoint selected by the caller. + pub checkpoint_id: String, + /// Current executable for the checkpoint's backend. + pub binary_path: PathBuf, +} + +/// Result of one completed checkpoint restore. +#[derive(Debug, Clone)] +pub struct RestoreSandboxResult { + /// Updated durable sandbox record. + pub instance: SandboxInstance, + /// Checkpoint now selected by the catalog HEAD. + pub checkpoint_id: String, +} + +impl SandboxManager { + /// Replace a running backend and rootfs from one verified checkpoint. + pub async fn restore(&self, id: Uuid, request: RestoreSandbox) -> Result { + validate_checkpoint_id(&request.checkpoint_id) + .map_err(|error| BlazeDaemonError::BadRequest(error.to_string()))?; + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has unfinished {} operation", + journal.kind + ))); + } + if instance.state != SandboxState::Running { + return Err(BlazeDaemonError::Conflict(format!( + "instance {id} is {}, expected running", + instance.state + ))); + } + + let target = self + .checkpoints + .verify_restore_target(id, &request.checkpoint_id) + .map_err(checkpoint_store_error)?; + if target.metadata.policy_name != instance.policy_name + || target.metadata.image_digest != instance.image_digest + || target.metadata.backend != instance.backend + { + return Err(BlazeDaemonError::Conflict(format!( + "checkpoint {} runtime identity does not match instance {id}", + request.checkpoint_id + ))); + } + if target.metadata.snapshot_kind != SnapshotKind::Full { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "checkpoint {} does not contain a full snapshot", + request.checkpoint_id + ))); + } + + let current_backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if current_backend.instance_id() != id || current_backend.backend() != instance.backend { + self.mark_recovery(id)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + self.require_restore_backend_live(id, ¤t_backend) + .await?; + if !self.storage.supports_checkpoint_restore() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} configured storage does not support checkpoint restore" + ))); + } + let spawner = self.spawner(target.metadata.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no restore adapter for {}", + target.metadata.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support checkpoint restore", + target.metadata.backend + )) + })?; + if capability.backend != target.metadata.backend + || capability.version != target.metadata.backend_version + || capability.snapshot_kind != target.metadata.snapshot_kind + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "checkpoint {} requires {} version {:?} {:?}, but the current adapter provides \ + {} version {:?} {:?}", + request.checkpoint_id, + target.metadata.backend, + target.metadata.backend_version, + target.metadata.snapshot_kind, + capability.backend, + capability.version, + capability.snapshot_kind + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + let expose_guest_socket = !current_backend.guest_socket_path().as_os_str().is_empty(); + + instance.begin_restore_operation(request.checkpoint_id.clone())?; + crate::failpoint::state("restore-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("restore-after-begin").await; + + let transaction = match crate::failpoint::storage("restore-storage-stage") { + Ok(()) => { + self.storage + .stage_checkpoint_restore(&storage, &target.rootfs_path) + .await + } + Err(error) => Err(error), + }; + let transaction = match transaction { + Ok(transaction) => transaction, + Err(error) => { + return Err(self + .fail_before_restore_stop(instance, None, error.into()) + .await); + } + }; + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageStaged) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-staged-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .fail_before_restore_stop(instance, Some(&transaction), error) + .await); + } + crate::failpoint::pause("restore-after-stage").await; + + let stopped = match crate::failpoint::backend("restore-backend-stop") { + Ok(()) => current_backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + let abort = self + .storage + .abort_checkpoint_restore(&transaction) + .await + .err(); + return Err(self.fail_after_restore_stop( + instance, + format!( + "current backend termination failed: {error}{}", + abort + .map(|error| format!("; staged storage cleanup failed: {error}")) + .unwrap_or_default() + ), + )); + } + + instance.backend_ownership = BackendOwnership::Stopped; + let stopped_state = instance + .advance_restore_phase(OperationPhase::RestoreBackendStopped) + .and_then(|_| instance.transition(SandboxState::Restoring)) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-stopped-state")?; + self.persist_and_retain(instance.clone()) + }); + if let Err(error) = stopped_state { + self.remove_backend_owner(id); + return Err(self.fail_after_restore_stop( + instance, + format!("backend stopped but lifecycle commit failed: {error}"), + )); + } + self.remove_backend_owner(id); + crate::failpoint::pause("restore-after-stop").await; + + let activated = match crate::failpoint::storage("restore-storage-activate") { + Ok(()) => self.storage.activate_checkpoint_restore(&transaction).await, + Err(error) => Err(error), + }; + if let Err(error) = activated { + let abort = self + .storage + .abort_checkpoint_restore(&transaction) + .await + .err(); + return Err(self.fail_after_restore_stop( + instance, + format!( + "replacement storage activation failed: {error}{}", + abort + .map(|error| format!("; predecessor restore failed: {error}")) + .unwrap_or_default() + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageActivated) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-activated-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage activated but lifecycle commit failed: {error}"), + )); + } + crate::failpoint::pause("restore-after-activate").await; + + let run_dir = self.runtime_dir(id); + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self.fail_after_restore_stop( + instance, + format!("prepare replacement backend ownership failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = crate::failpoint::state("restore-starting-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend intent commit failed: {error}"), + )); + } + + let restored = match crate::failpoint::backend("restore-backend-start") { + Ok(()) => { + spawner + .restore(RestoreRequest { + instance_id: id, + run_dir, + binary_path: request.binary_path, + storage, + snapshot_path: target.snapshot_path, + mem_path: target.memory_path, + checkpoint_backend: target.metadata.backend, + expected_version: target.metadata.backend_version.clone(), + snapshot_kind: target.metadata.snapshot_kind, + expose_guest_socket, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let restored = match restored { + Ok(owner) => owner, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + let _ = self.retain_backend(id, owner); + instance.backend_ownership = BackendOwnership::Running; + } else { + instance.backend_ownership = BackendOwnership::Stopped; + } + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend start failed: {source}"), + )); + } + }; + if let Some(error) = self.retain_backend(id, restored.clone()) { + instance.backend_ownership = BackendOwnership::Running; + return Err(self.fail_after_restore_stop(instance, error)); + } + instance.backend_ownership = BackendOwnership::Running; + + if restored.instance_id() != id + || restored.backend() != target.metadata.backend + || restored.version().map(str::to_string) != target.metadata.backend_version + { + return Err(self.fail_after_restore_stop( + instance, + format!( + "replacement backend identity ({}, {}, {:?}) does not match checkpoint \ + identity ({id}, {}, {:?})", + restored.instance_id(), + restored.backend(), + restored.version(), + target.metadata.backend, + target.metadata.backend_version + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreBackendStarted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-started-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend started but lifecycle commit failed: {error}"), + )); + } + if let Err(error) = self + .verify_restored_backend(id, &restored, expose_guest_socket) + .await + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement backend readiness failed: {error}"), + )); + } + + let head_updated = match crate::failpoint::storage("restore-head-update") { + Ok(()) => self + .checkpoints + .set_head(id, &request.checkpoint_id) + .map_err(checkpoint_store_error), + Err(error) => Err(error.into()), + }; + if let Err(error) = head_updated { + let observed = self.checkpoints.read_head(id); + return Err(self.fail_after_restore_stop( + instance, + format!( + "checkpoint HEAD update failed: {error}; observed HEAD after failure: \ + {observed:?}" + ), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreHeadUpdated) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-head-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("checkpoint HEAD changed but lifecycle commit failed: {error}"), + )); + } + crate::failpoint::pause("restore-after-head").await; + + let committed = match crate::failpoint::storage("restore-storage-commit") { + Ok(()) => self.storage.commit_checkpoint_restore(&transaction).await, + Err(error) => Err(error), + }; + if let Err(error) = committed { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage commit failed: {error}"), + )); + } + if let Err(error) = instance + .advance_restore_phase(OperationPhase::RestoreStorageCommitted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("restore-committed-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_after_restore_stop( + instance, + format!("replacement storage committed but lifecycle commit failed: {error}"), + )); + } + + let recovery_instance = instance.clone(); + instance.transition(SandboxState::Running)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("restore-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_after_restore_stop( + recovery_instance, + format!("replacement is live but final lifecycle commit failed: {error}"), + )); + } + Ok(RestoreSandboxResult { + instance, + checkpoint_id: request.checkpoint_id, + }) + } + + async fn require_restore_backend_live( + &self, + id: Uuid, + backend: &DynBackendInstance, + ) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited before restore \ + (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))) + } + Err(error) => { + self.mark_recovery(id)?; + Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))) + } + } + } + + async fn verify_restored_backend( + &self, + id: Uuid, + backend: &DynBackendInstance, + expose_guest_socket: bool, + ) -> Result<()> { + self.require_restore_backend_live(id, backend).await?; + if expose_guest_socket { + self.wait_for_guest_ready(backend, "restore-guest-ready") + .await?; + } + self.require_restore_backend_live(id, backend).await + } + + async fn fail_before_restore_stop( + &self, + mut instance: SandboxInstance, + transaction: Option<&StorageRestoreTransaction>, + original: BlazeDaemonError, + ) -> BlazeDaemonError { + let storage_cleanup = match transaction { + Some(transaction) => self + .storage + .abort_checkpoint_restore(transaction) + .await + .map_err(BlazeDaemonError::from), + None => self + .storage + .reconcile_checkpoint_restore(&instance.id.to_string()) + .await + .map_err(BlazeDaemonError::from), + }; + if let Err(cleanup) = storage_cleanup { + return self.fail_after_restore_stop( + instance, + format!("{original}; staged storage cleanup failed: {cleanup}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance.clone()) { + return self.fail_after_restore_stop( + instance, + format!("{original}; restore journal cleanup failed: {error}"), + ); + } + original + } + + fn fail_after_restore_stop( + &self, + instance: SandboxInstance, + cause: impl std::fmt::Display, + ) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "restore {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } +} + +fn checkpoint_store_error(error: impl std::fmt::Display) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!("checkpoint store: {error}")) +} diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index 6d9c4583b4..c8a51fd94b 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -14,7 +14,9 @@ use std::time::Duration; use std::time::Instant; use async_trait::async_trait; -use blaze_core::backend::{BackendKind, SnapshotRequest, SnapshotResult, SpawnRequest}; +use blaze_core::backend::{ + BackendKind, RestoreCapability, RestoreRequest, SnapshotRequest, SnapshotResult, SpawnRequest, +}; use blaze_core::guest_protocol::DEFAULT_MAX_RESPONSE_BYTES; use blaze_core::{BlazeError, Result}; use tokio::io::{AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncWriteExt, BufReader}; @@ -98,6 +100,9 @@ pub trait BackendInstance: Send + Sync { /// Shared backend instance handle stored in the daemon runtime map. pub type DynBackendInstance = Arc; +/// Restore outcome that preserves ownership when cleanup cannot be confirmed. +pub type RestoreResult = std::result::Result; + /// Backend start failure that may retain ownership of a started process. pub struct SpawnFailure { source: BlazeError, @@ -177,7 +182,7 @@ impl From for SpawnFailure { /// Factory for owned backend runtime instances. #[async_trait] pub trait BackendSpawner: Send + Sync { - /// Persist backend-specific pre-spawn ownership metadata. + /// Persist backend-specific ownership metadata before spawn or restore. async fn prepare_spawn(&self, _run_dir: &Path) -> Result<()> { Ok(()) } @@ -188,6 +193,26 @@ pub trait BackendSpawner: Send + Sync { request: SpawnRequest, ) -> std::result::Result; + /// Report the restore identity of the requested backend executable. + /// + /// `None` means restore is unsupported. Implementations that return a + /// version must inspect `binary_path` for every call rather than reusing + /// mutable process-wide state. + async fn restore_capability(&self, _binary_path: &Path) -> Result> { + Ok(None) + } + + /// Start an owned backend from committed checkpoint artifacts. + /// + /// Callers prepare the PID handoff through [`Self::prepare_spawn`] first. + /// Failures transfer any owner whose cleanup could not be confirmed. + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let _ = request; + Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "checkpoint restore is not supported by this backend".to_string(), + })) + } + /// Probe whether the configured backend executable is usable. async fn probe(&self, binary_path: &Path) -> Result; @@ -369,6 +394,75 @@ impl BackendSpawner for MockSpawner { .map_err(SpawnFailure::from) } + async fn restore_capability(&self, _binary_path: &Path) -> Result> { + Ok(Some(RestoreCapability { + backend: BackendKind::Mock, + version: Some("mock-v1".to_string()), + snapshot_kind: blaze_core::backend::SnapshotKind::Full, + })) + } + + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let RestoreRequest { + instance_id, + run_dir, + snapshot_path, + mem_path, + checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + .. + } = request; + if checkpoint_backend != BackendKind::Mock + || expected_version.as_deref() != Some("mock-v1") + || snapshot_kind != blaze_core::backend::SnapshotKind::Full + { + return Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "mock checkpoint identity is incompatible with the restore adapter" + .to_string(), + })); + } + let vmstate: serde_json::Value = match tokio::fs::read(&snapshot_path) + .await + .map_err(BlazeError::from) + .and_then(|bytes| { + serde_json::from_slice(&bytes).map_err(|error| BlazeError::BackendError { + msg: format!("decode mock VM state: {error}"), + }) + }) { + Ok(vmstate) => vmstate, + Err(error) => return Err(SpawnFailure::clean(error)), + }; + if vmstate.get("format").and_then(serde_json::Value::as_str) != Some("blaze-mock-v1") + || vmstate + .get("instance_id") + .and_then(serde_json::Value::as_str) + != Some(instance_id.to_string().as_str()) + || vmstate.get("kind").and_then(serde_json::Value::as_str) != Some("full") + { + return Err(SpawnFailure::clean(BlazeError::BackendError { + msg: "mock VM state does not match the requested sandbox".to_string(), + })); + } + let files = match tokio::fs::read(&mem_path) + .await + .map_err(BlazeError::from) + .and_then(|bytes| { + serde_json::from_slice::>>(&bytes).map_err(|error| { + BlazeError::BackendError { + msg: format!("decode mock guest memory: {error}"), + } + }) + }) { + Ok(files) => files, + Err(error) => return Err(SpawnFailure::clean(error)), + }; + spawn_mock_instance_with_files(instance_id, run_dir, files, expose_guest_socket) + .await + .map_err(SpawnFailure::from) + } + async fn probe(&self, _binary_path: &Path) -> Result { Ok(true) } @@ -389,37 +483,54 @@ struct MockInstance { } async fn spawn_mock_instance(instance_id: Uuid, run_dir: PathBuf) -> Result { + spawn_mock_instance_with_files(instance_id, run_dir, HashMap::new(), true).await +} + +async fn spawn_mock_instance_with_files( + instance_id: Uuid, + run_dir: PathBuf, + restored_files: HashMap>, + expose_guest_socket: bool, +) -> Result { tokio::fs::create_dir_all(&run_dir).await?; let socket = run_dir.join("vsock.uds"); if socket.exists() { tokio::fs::remove_file(&socket).await?; } - let listener = UnixListener::bind(&socket)?; let cancellation = CancellationToken::new(); let task_token = cancellation.clone(); - let files = Arc::new(Mutex::new(HashMap::new())); + let files = Arc::new(Mutex::new(restored_files)); let task_files = files.clone(); - let task = tokio::spawn(async move { - loop { - tokio::select! { - _ = task_token.cancelled() => break, - accepted = listener.accept() => { - let Ok((stream, _)) = accepted else { - break; - }; - let files = task_files.clone(); - tokio::spawn(async move { - if let Err(error) = serve_mock_guest(stream, files).await { - tracing::debug!(%error, "mock guest connection ended"); - } - }); + let (guest_socket_path, task) = if expose_guest_socket { + let listener = UnixListener::bind(&socket)?; + let task = tokio::spawn(async move { + loop { + tokio::select! { + _ = task_token.cancelled() => break, + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { + break; + }; + let files = task_files.clone(); + tokio::spawn(async move { + if let Err(error) = serve_mock_guest(stream, files).await { + tracing::debug!(%error, "mock guest connection ended"); + } + }); + } } } - } - }); + }); + (socket, task) + } else { + let task = tokio::spawn(async move { + task_token.cancelled().await; + }); + (PathBuf::new(), task) + }; Ok(Arc::new(MockInstance { instance_id, - guest_socket_path: socket, + guest_socket_path, cancellation, task: Mutex::new(Some(task)), files, @@ -1024,7 +1135,7 @@ mod tests { #[cfg(target_os = "linux")] use std::time::Duration; - use blaze_core::backend::{SnapshotKind, SnapshotRequest, SpawnRequest}; + use blaze_core::backend::{RestoreRequest, SnapshotKind, SnapshotRequest, SpawnRequest}; use blaze_core::policy::BackendConfigs; use blaze_core::storage::StorageSlot; @@ -1135,6 +1246,39 @@ mod tests { assert!(instance.snapshot(request).await.is_err()); } + #[tokio::test] + async fn checkpoint_restore_defaults_fail_closed_without_an_owner() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = request(temp.path()); + let restore = RestoreRequest { + instance_id: spawn.instance_id, + run_dir: spawn.run_dir, + binary_path: spawn.binary_path, + storage: spawn.storage, + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + checkpoint_backend: BackendKind::Bubblewrap, + expected_version: None, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + }; + + assert!( + BubblewrapSpawner + .restore_capability(Path::new("")) + .await + .expect("capability") + .is_none() + ); + let failure = match BubblewrapSpawner.restore(restore).await { + Ok(_) => panic!("restore must remain unsupported"), + Err(failure) => failure, + }; + let (source, owner) = failure.into_parts(); + assert!(source.to_string().contains("restore is not supported")); + assert!(owner.is_none()); + } + #[tokio::test] async fn mock_instance_captures_self_contained_state() { let temp = tempfile::tempdir().expect("temp"); diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index 7d30b0240b..e992b2a32f 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -9,7 +9,8 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use blaze_core::backend::{ - BackendKind, SnapshotKind, SnapshotRequest, SnapshotResult, SpawnRequest, + BackendKind, RestoreCapability, RestoreRequest, SnapshotKind, SnapshotRequest, SnapshotResult, + SpawnRequest, }; use blaze_core::policy::{FirecrackerConfig, VmConfig, parse_memory_value, to_mib_ceil}; use blaze_core::{BlazeError, Result}; @@ -26,7 +27,7 @@ use uuid::Uuid; #[cfg(target_os = "linux")] use super::terminate_recorded_process; use super::{ - BackendInstance, BackendSpawner, DynBackendInstance, SpawnFailure, SpawnResult, + BackendInstance, BackendSpawner, DynBackendInstance, RestoreResult, SpawnFailure, SpawnResult, configure_pid_handoff, prepare_pid_handoff, record_backend_stopped, remove_file_if_exists, spawn_result, stopped_marker, terminate_child, }; @@ -66,15 +67,25 @@ impl FirecrackerSpawner { async fn start( &self, request: SpawnRequest, + restore: Option, ) -> std::result::Result { validate_regular_file(&request.binary_path, "firecracker binary")?; validate_regular_file(&request.storage.rootfs_path, "rootfs")?; - validate_regular_file(&self.images_dir.join("vmlinux"), "vmlinux")?; - tokio::fs::create_dir_all(&request.run_dir).await?; + match &restore { + Some(restore) => { + validate_regular_file(&restore.snapshot_path, "VM-state snapshot")?; + validate_regular_file(&restore.mem_path, "memory snapshot")?; + } + None => validate_regular_file(&self.images_dir.join("vmlinux"), "vmlinux")?, + } let api_socket = request.run_dir.join("api.sock"); let capture = self .capture_for(&request.binary_path, api_socket.clone()) .await?; + if let Some(restore) = &restore { + validate_restore_compatibility(restore, &capture.backend_version)?; + } + tokio::fs::create_dir_all(&request.run_dir).await?; let guest_socket = request.run_dir.join("vsock.uds"); let pid_file = request.run_dir.join("firecracker.pid"); let stopped_marker = stopped_marker(&request.run_dir); @@ -87,9 +98,15 @@ impl FirecrackerSpawner { .as_ref() .cloned() .unwrap_or_default(); + let expose_guest_socket = restore.as_ref().map_or(fc_config.enable_vsock, |restore| { + restore.expose_guest_socket + }); let mut command = build_launch_command(&request.binary_path, &api_socket); - let config_path = write_vm_config(&self.images_dir, &request, &fc_config, &guest_socket)?; - command.arg("--config-file").arg(config_path); + if restore.is_none() { + let config_path = + write_vm_config(&self.images_dir, &request, &fc_config, &guest_socket)?; + command.arg("--config-file").arg(config_path); + } configure_logs(&mut command, &request.run_dir, fc_config.serial_log)?; command.env("BLAZE_INSTANCE_ID", request.instance_id.to_string()); let pid_handoff = configure_pid_handoff(&mut command, &pid_file)?; @@ -105,23 +122,29 @@ impl FirecrackerSpawner { child, capture, guest_socket, - fc_config.enable_vsock, + expose_guest_socket, pid_file, stopped_marker, )); return Err(SpawnFailure::compensate_started(error, owner).await); } - let instance = FirecrackerInstance::new( + let instance = Arc::new(FirecrackerInstance::new( request.instance_id, child, capture, guest_socket, - fc_config.enable_vsock, + expose_guest_socket, pid_file, stopped_marker, - ); - Ok(Arc::new(instance)) + )); + if let Some(restore) = restore + && let Err(error) = instance.load_snapshot(&restore).await + { + let owner: DynBackendInstance = instance; + return Err(SpawnFailure::compensate_started(error, owner).await); + } + Ok(instance) } } @@ -136,7 +159,53 @@ impl BackendSpawner for FirecrackerSpawner { &self, request: SpawnRequest, ) -> std::result::Result { - self.start(request).await + self.start(request, None).await + } + + async fn restore_capability(&self, binary_path: &Path) -> Result> { + validate_regular_file(binary_path, "firecracker binary")?; + Ok(Some(RestoreCapability { + backend: BackendKind::Firecracker, + version: Some(read_backend_version(binary_path).await?), + snapshot_kind: SnapshotKind::Full, + })) + } + + async fn restore(&self, request: RestoreRequest) -> RestoreResult { + let RestoreRequest { + instance_id, + run_dir, + binary_path, + storage, + snapshot_path, + mem_path, + checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + } = request; + self.start( + SpawnRequest { + instance_id, + run_dir, + binary_path, + storage, + // Snapshot restore does not reconstruct the original policy. + // Keep host-side logging disabled unless a future restore + // contract models that setting explicitly. + backend: blaze_core::policy::BackendConfigs::default(), + vm: None, + }, + Some(FirecrackerRestore { + snapshot_path, + mem_path, + backend: checkpoint_backend, + expected_version, + snapshot_kind, + expose_guest_socket, + }), + ) + .await } async fn probe(&self, binary_path: &Path) -> Result { @@ -187,6 +256,10 @@ impl FirecrackerInstance { killed: AtomicBool::new(false), } } + + async fn load_snapshot(&self, restore: &FirecrackerRestore) -> Result<()> { + self.capture.api.load_snapshot(restore).await + } } #[async_trait] @@ -313,6 +386,16 @@ struct FirecrackerCapture { backend_version: String, } +#[derive(Debug)] +struct FirecrackerRestore { + snapshot_path: PathBuf, + mem_path: PathBuf, + backend: BackendKind, + expected_version: Option, + snapshot_kind: SnapshotKind, + expose_guest_socket: bool, +} + impl FirecrackerCapture { fn new(api_socket: PathBuf, api_timeout: Duration, backend_version: String) -> Self { Self { @@ -333,6 +416,23 @@ impl FirecrackerApiClient { Self { socket, timeout } } + async fn load_snapshot(&self, restore: &FirecrackerRestore) -> Result<()> { + self.call_json( + Method::PUT, + "/snapshot/load", + Some(serde_json::json!({ + "snapshot_path": path_string(&restore.snapshot_path, "VM-state snapshot")?, + "mem_backend": { + "backend_type": "File", + "backend_path": path_string(&restore.mem_path, "memory snapshot")? + }, + "resume_vm": true + })), + ) + .await?; + Ok(()) + } + async fn call_json( &self, method: Method, @@ -411,6 +511,41 @@ impl FirecrackerApiClient { } } +fn validate_restore_compatibility( + restore: &FirecrackerRestore, + actual_version: &str, +) -> Result<()> { + if restore.backend != BackendKind::Firecracker { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker cannot restore a {} checkpoint", + restore.backend + ), + }); + } + if restore.snapshot_kind != SnapshotKind::Full { + return Err(BlazeError::BackendError { + msg: "Firecracker restore accepts only full checkpoints".to_string(), + }); + } + let expected_version = + restore + .expected_version + .as_deref() + .ok_or_else(|| BlazeError::BackendError { + msg: "Firecracker restore requires a checkpoint backend version".to_string(), + })?; + if expected_version != actual_version { + return Err(BlazeError::BackendError { + msg: format!( + "Firecracker checkpoint version {expected_version:?} does not match \ + executable version {actual_version:?}" + ), + }); + } + Ok(()) +} + impl FirecrackerInstance { async fn cleanup(&self) -> Result<()> { remove_if_exists(&self.capture.api.socket).await?; @@ -734,6 +869,9 @@ mod tests { #[cfg(target_os = "linux")] use tokio::sync::oneshot; + #[cfg(target_os = "linux")] + use crate::spawner::SpawnerRegistry; + use super::*; #[test] @@ -753,6 +891,41 @@ mod tests { assert!(parse_backend_version(b"Firecracker v1.15.0\nFirecracker v1.16.0\n").is_err()); } + #[test] + fn restore_compatibility_requires_the_matching_firecracker_version() { + let mut restore = FirecrackerRestore { + snapshot_path: PathBuf::from("vmstate.snap"), + mem_path: PathBuf::from("memory.snap"), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + }; + + validate_restore_compatibility(&restore, "Firecracker v1.16.0").expect("matching version"); + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.17.0") + .expect_err("mismatched version") + .to_string() + .contains("does not match executable version") + ); + restore.expected_version = None; + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.16.0") + .expect_err("missing version") + .to_string() + .contains("requires a checkpoint backend version") + ); + restore.expected_version = Some("Firecracker v1.16.0".to_string()); + restore.backend = BackendKind::Mock; + assert!( + validate_restore_compatibility(&restore, "Firecracker v1.16.0") + .expect_err("wrong backend") + .to_string() + .contains("cannot restore a mock checkpoint") + ); + } + #[cfg(target_os = "linux")] async fn spawn_api( socket: &Path, @@ -890,6 +1063,276 @@ mod tests { assert!(spawner.probe(&invalid).await.expect("replaced probe")); } + #[cfg(target_os = "linux")] + #[tokio::test] + async fn registry_restore_capability_reads_each_requested_binary() { + let temp = tempfile::tempdir().expect("temp"); + let first = temp.path().join("first"); + let second = temp.path().join("second"); + write_version_binary(&first, "Firecracker v1.15.0"); + write_version_binary(&second, "Firecracker v1.16.0"); + let mut registry = SpawnerRegistry::new(); + registry.insert( + BackendKind::Firecracker, + Arc::new(FirecrackerSpawner::new(temp.path().join("images"))), + ); + let adapter = registry + .get(BackendKind::Firecracker) + .expect("registered Firecracker adapter"); + + let first_capability = adapter + .restore_capability(&first) + .await + .expect("first capability") + .expect("restore supported"); + let second_capability = adapter + .restore_capability(&second) + .await + .expect("second capability") + .expect("restore supported"); + assert_eq!(first_capability.backend, BackendKind::Firecracker); + assert_eq!( + first_capability.version.as_deref(), + Some("Firecracker v1.15.0") + ); + assert_eq!(first_capability.snapshot_kind, SnapshotKind::Full); + assert_eq!( + second_capability.version.as_deref(), + Some("Firecracker v1.16.0") + ); + + write_version_binary(&first, "Firecracker v1.17.0"); + let replaced_capability = adapter + .restore_capability(&first) + .await + .expect("replaced capability") + .expect("restore supported"); + assert_eq!( + replaced_capability.version.as_deref(), + Some("Firecracker v1.17.0") + ); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn restore_rejects_a_binary_version_change_before_start() { + let temp = tempfile::tempdir().expect("temp"); + let spawn = spawn_request(temp.path()); + let run_dir = spawn.run_dir.clone(); + std::fs::remove_dir_all(&run_dir).expect("remove fixture run dir"); + write_version_binary(&spawn.binary_path, "Firecracker v1.17.0"); + std::fs::create_dir_all(spawn.storage.rootfs_path.parent().expect("rootfs parent")) + .expect("slot"); + std::fs::write(&spawn.storage.rootfs_path, b"rootfs").expect("rootfs"); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + std::fs::write(&snapshot_path, b"vmstate").expect("VM state"); + std::fs::write(&mem_path, b"memory").expect("memory"); + let spawner = FirecrackerSpawner::new(temp.path().join("images")); + + let failure = match spawner + .restore(RestoreRequest { + instance_id: spawn.instance_id, + run_dir: spawn.run_dir, + binary_path: spawn.binary_path, + storage: spawn.storage, + snapshot_path, + mem_path, + checkpoint_backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + }) + .await + { + Ok(_) => panic!("version change must fail before process start"), + Err(failure) => failure, + }; + let (source, owner) = failure.into_parts(); + + assert!( + source + .to_string() + .contains("does not match executable version") + ); + assert!(owner.is_none()); + assert!(!run_dir.exists()); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn instance_loads_a_full_snapshot_with_a_minimal_payload() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 1).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + child, + FirecrackerCapture::new( + api_socket, + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + PathBuf::new(), + false, + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + ); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + + instance + .load_snapshot(&FirecrackerRestore { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + }) + .await + .expect("load snapshot"); + + let calls = observed.await.expect("observed call"); + assert_eq!( + calls, + vec![( + Method::PUT, + "/snapshot/load".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_backend": { + "backend_type": "File", + "backend_path": mem_path, + }, + "resume_vm": true, + }), + )] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn restored_owner_exposes_the_guest_socket_when_requested() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let observed = spawn_api(&api_socket, 1).await; + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let guest_socket = temp.path().join("vsock.uds"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + child, + FirecrackerCapture::new( + api_socket, + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + guest_socket.clone(), + true, + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + ); + let snapshot_path = temp.path().join("vmstate.snap"); + let mem_path = temp.path().join("memory.snap"); + + assert_eq!(instance.guest_socket_path(), guest_socket); + instance + .load_snapshot(&FirecrackerRestore { + snapshot_path: snapshot_path.clone(), + mem_path: mem_path.clone(), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + }) + .await + .expect("load snapshot"); + + let calls = observed.await.expect("observed call"); + assert_eq!( + calls, + vec![( + Method::PUT, + "/snapshot/load".to_string(), + serde_json::json!({ + "snapshot_path": snapshot_path, + "mem_backend": { + "backend_type": "File", + "backend_path": mem_path, + }, + "resume_vm": true, + }), + )] + ); + instance.kill().await.expect("kill"); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn failed_snapshot_load_retains_an_owner_when_cleanup_is_incomplete() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let server = spawn_api_response( + &api_socket, + hyper::StatusCode::BAD_REQUEST, + b"incompatible snapshot".to_vec(), + Duration::ZERO, + ); + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let guest_socket = temp.path().join("guest.sock"); + std::fs::create_dir(&guest_socket).expect("cleanup blocker"); + let instance_id = Uuid::new_v4(); + let instance = Arc::new(FirecrackerInstance::new( + instance_id, + child, + FirecrackerCapture::new( + api_socket, + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + guest_socket.clone(), + true, + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + )); + let restore = FirecrackerRestore { + snapshot_path: temp.path().join("vmstate.snap"), + mem_path: temp.path().join("memory.snap"), + backend: BackendKind::Firecracker, + expected_version: Some("Firecracker v1.16.0".to_string()), + snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + }; + + let load_error = instance + .load_snapshot(&restore) + .await + .expect_err("snapshot load must fail"); + server.await.expect("server"); + let owner: DynBackendInstance = instance; + let failure = SpawnFailure::compensate_started(load_error, owner).await; + let (source, retained) = failure.into_parts(); + + assert!(source.to_string().contains("cleanup failed")); + let retained = retained.expect("incomplete cleanup must retain ownership"); + assert_eq!(retained.instance_id(), instance_id); + assert_eq!(retained.version(), Some("Firecracker v1.16.0")); + + std::fs::remove_dir(&guest_socket).expect("remove cleanup blocker"); + retained.kill().await.expect("retry cleanup"); + } + #[cfg(target_os = "linux")] #[tokio::test] async fn instance_reports_version_and_captures_full_snapshot_over_uds() { From 86c345e861301f982e4aa38e40d41eef12382a16 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 13:59:43 +0800 Subject: [PATCH 33/40] feat(blaze): hibernate and resume sandboxes This enables API users to stop a running sandbox after publishing a verified hibernation image, then resume it after daemon restart without losing guest-visible state. The manager checks capture and restore compatibility before journaling, hashes artifacts before publication, retains the storage slot while stopped, and owns a restored backend before readiness checks. Durable phases distinguish retryable failures from ownership that requires explicit cleanup. Tests cover API aliases, restart continuity, unsupported adapters, corrupt artifacts, and failures around snapshot, backend stop, publication, restore, readiness, and cleanup. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/lifecycle.rs | 217 +++- src/blaze/crates/blazed/src/api.rs | 561 ++++++++- src/blaze/crates/blazed/src/sandbox.rs | 2 + .../crates/blazed/src/sandbox/hibernate.rs | 1016 +++++++++++++++++ .../crates/blazed/src/sandbox/manager.rs | 80 +- 5 files changed, 1863 insertions(+), 13 deletions(-) create mode 100644 src/blaze/crates/blazed/src/sandbox/hibernate.rs diff --git a/src/blaze/crates/blaze-core/src/lifecycle.rs b/src/blaze/crates/blaze-core/src/lifecycle.rs index e2753a6224..33bb3704bd 100644 --- a/src/blaze/crates/blaze-core/src/lifecycle.rs +++ b/src/blaze/crates/blaze-core/src/lifecycle.rs @@ -24,6 +24,12 @@ pub enum SandboxState { Checkpointed, /// The previous backend is stopped while replacement resources are owned. Restoring, + /// A live backend is being converted into durable hibernation artifacts. + Hibernating, + /// Durable hibernation artifacts and storage are retained without a backend. + Hibernated, + /// A backend is being started from retained hibernation artifacts. + Resuming, RecoveryRequired, Reset, Warm, @@ -39,6 +45,9 @@ impl SandboxState { SandboxState::Paused => "paused", SandboxState::Checkpointed => "checkpointed", SandboxState::Restoring => "restoring", + SandboxState::Hibernating => "hibernating", + SandboxState::Hibernated => "hibernated", + SandboxState::Resuming => "resuming", SandboxState::RecoveryRequired => "recovery-required", SandboxState::Reset => "reset", SandboxState::Warm => "warm", @@ -57,6 +66,10 @@ pub enum OperationKind { Checkpoint, /// A running sandbox is being replaced from a selected checkpoint. Restore, + /// A live backend is being stopped after durable artifacts are prepared. + Hibernate, + /// A backend is being started from retained hibernation artifacts. + Resume, /// Runtime resources are being destroyed. Destroy, } @@ -67,6 +80,8 @@ impl OperationKind { OperationKind::Create => "create", OperationKind::Checkpoint => "checkpoint", OperationKind::Restore => "restore", + OperationKind::Hibernate => "hibernate", + OperationKind::Resume => "resume", OperationKind::Destroy => "destroy", } } @@ -108,6 +123,24 @@ pub enum OperationPhase { RestoreHeadUpdated, /// The storage replacement is committed and can no longer be aborted. RestoreStorageCommitted, + /// Hibernate intent is durable, but the backend is still running. + HibernatePreparing, + /// The backend is paused while hibernation artifacts are written. + HibernatePaused, + /// Hibernation artifacts are complete and durable. + HibernateArtifactsSynced, + /// The live backend has stopped and no longer owns runtime resources. + HibernateBackendStopped, + /// The replacement hibernation directory is durably visible. + HibernatePublished, + /// Resume intent is durable and no backend has started. + ResumePreparing, + /// Backend ownership intent is durable before restore starts. + ResumeBackendStarting, + /// A restored backend is owned, but readiness is not yet confirmed. + ResumeBackendStarted, + /// The restored backend and optional guest transport are ready. + ResumeBackendReady, } impl OperationPhase { @@ -124,6 +157,15 @@ impl OperationPhase { OperationPhase::RestoreBackendStarted => "restore-backend-started", OperationPhase::RestoreHeadUpdated => "restore-head-updated", OperationPhase::RestoreStorageCommitted => "restore-storage-committed", + OperationPhase::HibernatePreparing => "hibernate-preparing", + OperationPhase::HibernatePaused => "hibernate-paused", + OperationPhase::HibernateArtifactsSynced => "hibernate-artifacts-synced", + OperationPhase::HibernateBackendStopped => "hibernate-backend-stopped", + OperationPhase::HibernatePublished => "hibernate-published", + OperationPhase::ResumePreparing => "resume-preparing", + OperationPhase::ResumeBackendStarting => "resume-backend-starting", + OperationPhase::ResumeBackendStarted => "resume-backend-started", + OperationPhase::ResumeBackendReady => "resume-backend-ready", } } @@ -140,6 +182,15 @@ impl OperationPhase { | OperationPhase::RestoreBackendStarted | OperationPhase::RestoreHeadUpdated | OperationPhase::RestoreStorageCommitted => OperationKind::Restore, + OperationPhase::HibernatePreparing + | OperationPhase::HibernatePaused + | OperationPhase::HibernateArtifactsSynced + | OperationPhase::HibernateBackendStopped + | OperationPhase::HibernatePublished => OperationKind::Hibernate, + OperationPhase::ResumePreparing + | OperationPhase::ResumeBackendStarting + | OperationPhase::ResumeBackendStarted + | OperationPhase::ResumeBackendReady => OperationKind::Resume, } } @@ -156,6 +207,15 @@ impl OperationPhase { OperationPhase::RestoreBackendStarted => 4, OperationPhase::RestoreHeadUpdated => 5, OperationPhase::RestoreStorageCommitted => 6, + OperationPhase::HibernatePreparing => 0, + OperationPhase::HibernatePaused => 1, + OperationPhase::HibernateArtifactsSynced => 2, + OperationPhase::HibernateBackendStopped => 3, + OperationPhase::HibernatePublished => 4, + OperationPhase::ResumePreparing => 0, + OperationPhase::ResumeBackendStarting => 1, + OperationPhase::ResumeBackendStarted => 2, + OperationPhase::ResumeBackendReady => 3, } } } @@ -311,6 +371,40 @@ impl SandboxInstance { self.advance_operation_phase(OperationKind::Restore, phase) } + /// Record hibernation intent before pausing the backend. + pub fn begin_hibernate_operation(&mut self) -> Result<()> { + self.begin_operation(OperationKind::Hibernate)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.phase = Some(OperationPhase::HibernatePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active hibernation journal without replacing its identity. + pub fn advance_hibernate_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Hibernate, phase) + } + + /// Record resume intent before preparing a replacement backend. + pub fn begin_resume_operation(&mut self) -> Result<()> { + self.begin_operation(OperationKind::Resume)?; + let operation = self + .operation + .as_mut() + .expect("begin_operation installs a journal"); + operation.phase = Some(OperationPhase::ResumePreparing); + self.updated_at = Utc::now(); + Ok(()) + } + + /// Advance the active resume journal without replacing its identity. + pub fn advance_resume_phase(&mut self, phase: OperationPhase) -> Result<()> { + self.advance_operation_phase(OperationKind::Resume, phase) + } + fn advance_operation_phase( &mut self, requested_kind: OperationKind, @@ -388,7 +482,52 @@ impl SandboxInstance { } _ => true, }; - if !restore_boundary_reached || !is_valid_transition(self.state, target) { + let hibernate_boundary_reached = match (self.state, target) { + (SandboxState::Running, SandboxState::Hibernating) => self.operation_phase_reached( + OperationKind::Hibernate, + OperationPhase::HibernatePreparing, + ), + (SandboxState::Hibernating, SandboxState::Hibernated) => { + self.backend_ownership == BackendOwnership::Stopped + && self.operation_phase_reached( + OperationKind::Hibernate, + OperationPhase::HibernatePublished, + ) + } + (SandboxState::Hibernating, SandboxState::Running) => { + self.backend_ownership == BackendOwnership::Running + && self + .operation + .as_ref() + .is_some_and(|operation| operation.kind == OperationKind::Hibernate) + } + (SandboxState::Hibernated, SandboxState::Resuming) => { + self.backend_ownership == BackendOwnership::Stopped + && self.operation_phase_reached( + OperationKind::Resume, + OperationPhase::ResumePreparing, + ) + } + (SandboxState::Resuming, SandboxState::Running) => { + self.backend_ownership == BackendOwnership::Running + && self.operation_phase_reached( + OperationKind::Resume, + OperationPhase::ResumeBackendReady, + ) + } + (SandboxState::Resuming, SandboxState::Hibernated) => { + self.backend_ownership == BackendOwnership::Stopped + && self + .operation + .as_ref() + .is_some_and(|operation| operation.kind == OperationKind::Resume) + } + _ => true, + }; + if !restore_boundary_reached + || !hibernate_boundary_reached + || !is_valid_transition(self.state, target) + { return Err(BlazeError::InvalidStateTransition { from: self.state.to_string(), to: target.to_string(), @@ -418,12 +557,19 @@ impl SandboxInstance { } fn restore_phase_reached(&self, minimum: OperationPhase) -> bool { - minimum.operation_kind() == OperationKind::Restore + self.operation_phase_reached(OperationKind::Restore, minimum) + } + + fn operation_phase_reached( + &self, + operation_kind: OperationKind, + minimum: OperationPhase, + ) -> bool { + minimum.operation_kind() == operation_kind && self.operation.as_ref().is_some_and(|operation| { - operation.kind == OperationKind::Restore + operation.kind == operation_kind && operation.phase.is_some_and(|phase| { - phase.operation_kind() == OperationKind::Restore - && phase.rank() >= minimum.rank() + phase.operation_kind() == operation_kind && phase.rank() >= minimum.rank() }) }) } @@ -520,8 +666,8 @@ impl SandboxInstance { fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { use SandboxState::{ - Checkpointed, Creating, Destroyed, Paused, Pending, RecoveryRequired, Reset, Restoring, - Running, Warm, + Checkpointed, Creating, Destroyed, Hibernated, Hibernating, Paused, Pending, + RecoveryRequired, Reset, Restoring, Resuming, Running, Warm, }; if to == Destroyed { // `* → destroyed` is always valid (terminal sink). @@ -540,6 +686,12 @@ fn is_valid_transition(from: SandboxState, to: SandboxState) -> bool { (Checkpointed, Running) => true, (Running, Restoring) => true, (Restoring, Running) => true, + (Running, Hibernating) => true, + (Hibernating, Hibernated) => true, + (Hibernating, Running) => true, + (Hibernated, Resuming) => true, + (Resuming, Running) => true, + (Resuming, Hibernated) => true, (Reset, Warm) => true, (Warm, Creating) => true, // pool reuse / warm path _ => false, @@ -633,6 +785,57 @@ mod tests { .expect("restore commits"); } + #[test] + fn hibernation_state_requires_durable_ownership_boundaries() { + let mut inst = fresh(); + inst.transition(SandboxState::Creating).expect("creating"); + inst.transition(SandboxState::Running).expect("running"); + inst.backend_ownership = BackendOwnership::Running; + + let error = inst + .transition(SandboxState::Hibernating) + .expect_err("hibernate requires a journal"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.begin_hibernate_operation().expect("begin hibernate"); + inst.transition(SandboxState::Hibernating) + .expect("hibernate starts"); + inst.advance_hibernate_phase(OperationPhase::HibernateArtifactsSynced) + .expect("artifacts durable"); + let error = inst + .transition(SandboxState::Hibernated) + .expect_err("a live backend prevents hibernated state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.backend_ownership = BackendOwnership::Stopped; + inst.advance_hibernate_phase(OperationPhase::HibernatePublished) + .expect("publish hibernation"); + inst.transition(SandboxState::Hibernated) + .expect("hibernate commits"); + inst.finish_operation(); + + let error = inst + .transition(SandboxState::Resuming) + .expect_err("resume requires a journal"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.begin_resume_operation().expect("begin resume"); + inst.transition(SandboxState::Resuming) + .expect("resume starts"); + inst.advance_resume_phase(OperationPhase::ResumeBackendStarted) + .expect("backend started"); + inst.backend_ownership = BackendOwnership::Running; + let error = inst + .transition(SandboxState::Running) + .expect_err("readiness must precede running state"); + assert!(matches!(error, BlazeError::InvalidStateTransition { .. })); + + inst.advance_resume_phase(OperationPhase::ResumeBackendReady) + .expect("backend ready"); + inst.transition(SandboxState::Running) + .expect("resume commits"); + } + #[test] fn destroy_is_always_legal_except_from_destroyed() { let mut inst = fresh(); diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index f5a9096c0b..a96d9a8062 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -31,7 +31,9 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::guest::MAX_GUEST_FILE_BYTES; use crate::request_body; -use crate::sandbox::{CreateSandbox, RestoreSandbox, RestoreSandboxResult}; +use crate::sandbox::{ + CreateSandbox, HibernateSandbox, RestoreSandbox, RestoreSandboxResult, ResumeSandbox, +}; use crate::state::ServerState; const MAX_EXEC_TIMEOUT_SECS: u32 = 20; @@ -126,6 +128,10 @@ async fn dispatch( | ("POST", ["v1", "sandboxes", id, "rollback", checkpoint_id]) => { rollback(state, id, checkpoint_id).await } + ("POST", ["v1", "instances", id, "hibernate"]) + | ("POST", ["v1", "sandboxes", id, "hibernate"]) => hibernate(state, id).await, + ("POST", ["v1", "instances", id, "resume"]) + | ("POST", ["v1", "sandboxes", id, "resume"]) => resume(state, id).await, ("POST", ["v1", "instances", id, "reset"]) => reset_instance(state, id).await, ("DELETE", ["v1", "instances", id]) | ("DELETE", ["v1", "sandboxes", id]) @@ -349,6 +355,44 @@ async fn rollback( })) } +async fn hibernate(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = configured_backend_path(state, instance.backend)?; + json_ok( + &state + .manager + .hibernate(uuid, HibernateSandbox { binary_path }) + .await?, + ) +} + +async fn resume(state: &Arc, id: &str) -> Result>> { + let uuid = parse_uuid(id)?; + let instance = state.manager.get(uuid)?; + let binary_path = configured_backend_path(state, instance.backend)?; + json_ok( + &state + .manager + .resume(uuid, ResumeSandbox { binary_path }) + .await?, + ) +} + +fn configured_backend_path( + state: &ServerState, + backend: BackendKind, +) -> Result { + Ok(state + .config + .lock() + .map_err(|_| BlazeDaemonError::Internal("config lock poisoned".into()))? + .backends + .get(backend.as_str()) + .cloned() + .unwrap_or_default()) +} + async fn reset_instance(state: &Arc, id: &str) -> Result>> { let uuid = parse_uuid(id)?; let _operation = state @@ -2094,6 +2138,521 @@ mod tests { assert!(state.manager.backend_owner(uuid).is_some()); } + #[tokio::test] + async fn hibernate_releases_the_backend_and_resume_survives_restart() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .write_file(uuid, "/tmp/value".to_string(), b"hibernate-memory") + .await + .expect("write guest state"); + + let (status, hibernated) = dispatched_json( + &state, + Method::POST, + &format!("/v1/sandboxes/{id}/hibernate"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(hibernated["state"], "hibernated"); + assert_eq!(hibernated["backend_ownership"], "stopped"); + assert!(state.manager.backend_owner(uuid).is_none()); + let hibernate_dir = config.daemon.state_dir.join(id).join("hibernate"); + for name in ["manifest.json", "memory.snap", "vmstate.snap"] { + assert!(hibernate_dir.join(name).is_file(), "{name} is missing"); + } + let report = state.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + drop(state); + + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let restarted = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + assert_eq!( + restarted.manager.get(uuid).expect("loaded state").state, + SandboxState::Hibernated + ); + let report = restarted.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + + let (status, resumed) = dispatched_json( + &restarted, + Method::POST, + &format!("/v1/instances/{id}/resume"), + Vec::new(), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert_eq!(resumed["state"], "running"); + assert_eq!( + restarted + .manager + .read_file(uuid, "/tmp/value".to_string()) + .await + .expect("read resumed guest state"), + b"hibernate-memory" + ); + assert!( + hibernate_dir.is_dir(), + "the last hibernation image remains available until replacement or destroy" + ); + assert!(restarted.manager.destroy(uuid).await.expect("destroy")); + assert!(!hibernate_dir.exists()); + } + + #[tokio::test] + async fn hibernate_rejects_a_capture_only_backend_before_state_mutation() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(CaptureOnlyMockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + + let error = state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("resume capability is required"); + + assert!(matches!(error, BlazeDaemonError::UnsupportedOperation(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert!(lifecycle.operation.is_none()); + } + + #[tokio::test] + async fn resume_rejects_corrupted_hibernation_artifacts_without_starting_a_backend() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = build_test_state( + config.clone(), + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + tokio::fs::write( + config + .daemon + .state_dir + .join(id) + .join("hibernate/memory.snap"), + b"corrupted", + ) + .await + .expect("corrupt artifact"); + + let error = state + .manager + .resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect_err("corrupted artifact must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert!(lifecycle.operation.is_none()); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[tokio::test] + async fn startup_retains_an_interrupted_hibernation_for_explicit_cleanup() { + let temp = tempfile::tempdir().expect("temp"); + let config = test_config(&temp); + let storage: Arc = Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let mut instance = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:ownership-test".into(), + StartPath::Cold, + "ownership-test".into(), + ); + instance + .transition(SandboxState::Creating) + .expect("creating"); + instance.transition(SandboxState::Running).expect("running"); + instance.backend_ownership = BackendOwnership::Running; + instance + .begin_hibernate_operation() + .expect("begin hibernation"); + instance + .transition(SandboxState::Hibernating) + .expect("hibernating"); + instance.persist(&config.daemon.state_dir).expect("persist"); + storage + .acquire(&AcquireOpts { + instance_id: instance.id.to_string(), + rootfs_size: 4096, + mem_size: 4096, + }) + .await + .expect("storage"); + let id = instance.id; + let state = build_test_state( + config, + test_policy(BackendKind::Mock, false), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ); + + let report = state.manager.reconcile_startup().await; + assert_eq!(report.attempted, 0); + assert!(report.failures.is_empty()); + let retained = state.manager.get(id).expect("retained lifecycle"); + assert_eq!(retained.state, SandboxState::RecoveryRequired); + assert_eq!( + retained.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate) + ); + assert!(state.manager.destroy(id).await.expect("explicit destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_snapshot_failure_resumes_the_existing_backend() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-snapshot"]); + + hook.run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("snapshot failure"); + + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Running); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Running); + assert!(lifecycle.operation.is_none()); + let names = std::fs::read_dir(state.state_dir.join(id)) + .expect("instance directory") + .map(|entry| entry.expect("entry").file_name()) + .collect::>(); + assert!( + names + .iter() + .all(|name| !name.to_string_lossy().starts_with(".hibernate.")) + ); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_compensation_requires_guest_readiness() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = + crate::failpoint::TestFailpoint::new(&["hibernate-snapshot", "resume-guest-ready"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("guest readiness must fail closed"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn uncertain_hibernate_stop_retains_the_existing_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let owner = state.manager.backend_owner(uuid).expect("owner"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-backend-stop"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("uncertain stop must retain ownership"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + let retained = state.manager.backend_owner(uuid).expect("retained owner"); + assert!(Arc::ptr_eq(&owner, &retained)); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::HibernateArtifactsSynced) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn hibernate_publish_failure_retains_stopped_ownership_for_destroy() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + let hook = crate::failpoint::TestFailpoint::new(&["hibernate-publish"]); + + let error = hook + .run(state.manager.hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("publish failure follows backend stop"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::HibernateBackendStopped) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_start_failure_preserves_retryable_hibernation() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = crate::failpoint::TestFailpoint::new(&["resume-backend-start"]); + + hook.run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("resume start failure"); + + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Hibernated); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert!(lifecycle.operation.is_none()); + state + .manager + .resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("retry resume"); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_readiness_failure_cleans_the_replacement_backend() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = crate::failpoint::TestFailpoint::new(&["resume-guest-ready"]); + + hook.run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("readiness failure"); + + assert!(state.manager.backend_owner(uuid).is_none()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::Hibernated); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Stopped); + assert!(lifecycle.operation.is_none()); + } + + #[cfg(feature = "test-failpoints")] + #[tokio::test] + async fn resume_cleanup_failure_retains_the_replacement_owner() { + let temp = tempfile::tempdir().expect("temp"); + let state = mock_state(&temp, false); + let created = created_json(&state, &test_request()).await; + let id = created["instance"]["id"].as_str().expect("id"); + let uuid = Uuid::parse_str(id).expect("uuid"); + state + .manager + .hibernate( + uuid, + HibernateSandbox { + binary_path: PathBuf::new(), + }, + ) + .await + .expect("hibernate"); + let hook = + crate::failpoint::TestFailpoint::new(&["resume-guest-ready", "resume-backend-stop"]); + + let error = hook + .run(state.manager.resume( + uuid, + ResumeSandbox { + binary_path: PathBuf::new(), + }, + )) + .await + .expect_err("failed cleanup must retain ownership"); + + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(state.manager.backend_owner(uuid).is_some()); + let lifecycle = state.manager.get(uuid).expect("lifecycle"); + assert_eq!(lifecycle.state, SandboxState::RecoveryRequired); + assert_eq!(lifecycle.backend_ownership, BackendOwnership::Unknown); + assert_eq!( + lifecycle.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Resume) + ); + assert_eq!( + lifecycle + .operation + .as_ref() + .and_then(|operation| operation.phase), + Some(OperationPhase::ResumeBackendStarted) + ); + assert!(state.manager.destroy(uuid).await.expect("destroy")); + } + #[tokio::test] async fn rollback_replaces_runtime_state_without_rewriting_capture_history() { let temp = tempfile::tempdir().expect("temp"); diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs index 42ad6248c2..479b111dc1 100644 --- a/src/blaze/crates/blazed/src/sandbox.rs +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -2,8 +2,10 @@ //! Managed sandbox lifecycle and runtime ownership. mod checkpoint; +mod hibernate; mod manager; mod restore; +pub use hibernate::{HibernateSandbox, ResumeSandbox}; pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; pub use restore::{RestoreSandbox, RestoreSandboxResult}; diff --git a/src/blaze/crates/blazed/src/sandbox/hibernate.rs b/src/blaze/crates/blazed/src/sandbox/hibernate.rs new file mode 100644 index 0000000000..3a01279041 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/hibernate.rs @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable hibernation and restartable resume for managed sandboxes. + +use std::collections::BTreeSet; +use std::future::Future; +use std::path::{Path, PathBuf}; +use std::pin::Pin; + +use blaze_core::backend::{BackendKind, RestoreRequest, SnapshotKind, SnapshotRequest}; +use blaze_core::checkpoint::CheckpointArtifact; +use blaze_core::lifecycle::{BackendOwnership, OperationPhase, SandboxInstance, SandboxState}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; +use crate::spawner::DynBackendInstance; + +use super::manager::SandboxManager; + +const HIBERNATE_FORMAT_VERSION: u32 = 1; +const HIBERNATE_DIRECTORY: &str = "hibernate"; +const MANIFEST_ARTIFACT: &str = "manifest.json"; +const MEMORY_ARTIFACT: &str = "memory.snap"; +const VMSTATE_ARTIFACT: &str = "vmstate.snap"; +const REQUIRED_ARTIFACTS: [&str; 2] = [VMSTATE_ARTIFACT, MEMORY_ARTIFACT]; + +/// Inputs resolved from the current daemon configuration before hibernation. +#[derive(Debug, Clone)] +pub struct HibernateSandbox { + /// Current executable for the sandbox backend. + pub binary_path: PathBuf, +} + +/// Inputs resolved from the current daemon configuration before resume. +#[derive(Debug, Clone)] +pub struct ResumeSandbox { + /// Current executable for the sandbox backend. + pub binary_path: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct HibernateManifest { + format_version: u32, + sandbox_id: Uuid, + policy_name: String, + image_digest: String, + backend: BackendKind, + backend_version: Option, + snapshot_kind: SnapshotKind, + expose_guest_socket: bool, + artifacts: Vec, +} + +impl SandboxManager { + /// Stop a running backend after publishing durable hibernation artifacts. + pub fn hibernate( + &self, + id: Uuid, + request: HibernateSandbox, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + require_quiescent_state(&instance, SandboxState::Running)?; + + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::Conflict(format!("instance {id} has no backend owner")) + })?; + if backend.instance_id() != id || backend.backend() != instance.backend { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend owner identity does not match durable state" + ))); + } + require_backend_live(id, &backend).await.map_err(|error| { + let recovery = self.mark_instance_recovery(instance.clone()).err(); + with_recovery_error(error, recovery) + })?; + if !backend.supports_checkpoint_capture() { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support hibernation", + backend.backend() + ))); + } + + let spawner = self.spawner(instance.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no resume adapter for {}", + instance.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support resume", + instance.backend + )) + })?; + let backend_version = backend.version().map(str::to_string); + if capability.backend != instance.backend + || capability.version != backend_version + || capability.snapshot_kind != SnapshotKind::Full + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend capture identity does not match its resume adapter" + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + let expose_guest_socket = !backend.guest_socket_path().as_os_str().is_empty(); + let hibernate_dir = self.hibernate_dir(id); + if let Err(error) = prepare_hibernate_directory(&hibernate_dir).await { + let recovery = self.mark_instance_recovery(instance).err(); + return Err(with_recovery_error(error, recovery)); + } + + instance.begin_hibernate_operation()?; + instance.transition(SandboxState::Hibernating)?; + crate::failpoint::state("hibernate-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("hibernate-after-begin").await; + + let paused = match crate::failpoint::backend("hibernate-pause") { + Ok(()) => backend.pause().await, + Err(error) => Err(error), + }; + if let Err(error) = paused { + return Err(self + .compensate_hibernate( + instance, + &backend, + None, + format!("backend pause failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernatePaused) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-paused-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .compensate_hibernate( + instance, + &backend, + None, + format!("paused-state commit failed: {error}"), + ) + .await); + } + + let staging_dir = self.hibernate_staging_dir(id); + if let Err(error) = tokio::fs::create_dir(&staging_dir).await { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("staging directory creation failed: {error}"), + ) + .await); + } + let snapshot_path = staging_dir.join(VMSTATE_ARTIFACT); + let memory_path = staging_dir.join(MEMORY_ARTIFACT); + let snapshot = match crate::failpoint::backend("hibernate-snapshot") { + Ok(()) => { + backend + .snapshot(SnapshotRequest { + snapshot_path: snapshot_path.clone(), + mem_path: memory_path.clone(), + kind: SnapshotKind::Full, + }) + .await + } + Err(error) => Err(error), + }; + match snapshot { + Ok(snapshot) + if snapshot.snapshot_path == snapshot_path + && snapshot.mem_path == memory_path => {} + Ok(snapshot) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!( + "backend returned unexpected hibernation artifacts ({}, {})", + snapshot.snapshot_path.display(), + snapshot.mem_path.display() + ), + ) + .await); + } + Err(error) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("snapshot capture failed: {error}"), + ) + .await); + } + } + let flushed = match crate::failpoint::storage("hibernate-storage-flush") { + Ok(()) => self.storage.flush_dirty(&storage).await, + Err(error) => Err(error), + }; + if let Err(error) = flushed { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("storage flush failed: {error}"), + ) + .await); + } + + let manifest = match build_hibernate_manifest( + &staging_dir, + &instance, + capability.version, + expose_guest_socket, + ) + .await + { + Ok(manifest) => manifest, + Err(error) => { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact hashing failed: {error}"), + ) + .await); + } + }; + if let Err(error) = write_and_sync_manifest(&staging_dir, &manifest).await { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact publication failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernateArtifactsSynced) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-artifacts-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self + .compensate_hibernate( + instance, + &backend, + Some(&staging_dir), + format!("artifact-state commit failed: {error}"), + ) + .await); + } + + let stopped = match crate::failpoint::backend("hibernate-backend-stop") { + Ok(()) => backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + return Err(self.fail_hibernate_after_stop( + instance, + format!("backend termination failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernateBackendStopped) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-stopped-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_hibernate_after_stop( + instance, + format!("backend stopped but lifecycle commit failed: {error}"), + )); + } + self.remove_backend_owner(id); + crate::failpoint::pause("hibernate-after-stop").await; + + let backup_dir = self.hibernate_backup_dir(id); + if hibernate_dir.exists() { + if let Err(error) = tokio::fs::rename(&hibernate_dir, &backup_dir).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("previous hibernation backup failed: {error}"), + )); + } + if let Err(error) = sync_directory(self.instance_dir(id)).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("previous hibernation backup sync failed: {error}"), + )); + } + } + let published = match crate::failpoint::storage("hibernate-publish") { + Ok(()) => tokio::fs::rename(&staging_dir, &hibernate_dir) + .await + .map_err(BlazeDaemonError::from), + Err(error) => Err(error.into()), + }; + if let Err(error) = published { + return Err(self.fail_hibernate_after_stop( + instance, + format!("hibernate directory publication failed: {error}"), + )); + } + if let Err(error) = sync_directory(self.instance_dir(id)).await { + return Err(self.fail_hibernate_after_stop( + instance, + format!("hibernate directory sync failed: {error}"), + )); + } + if let Err(error) = instance + .advance_hibernate_phase(OperationPhase::HibernatePublished) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("hibernate-published-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_hibernate_after_stop( + instance, + format!("published-state commit failed: {error}"), + )); + } + + let recovery = instance.clone(); + instance.transition(SandboxState::Hibernated)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("hibernate-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_hibernate_after_stop( + recovery, + format!("final hibernated-state commit failed: {error}"), + )); + } + if backup_dir.exists() + && let Err(error) = remove_directory_and_sync(&backup_dir).await + { + tracing::warn!( + instance = %id, + %error, + "obsolete hibernation backup retained for later cleanup" + ); + } + Ok(instance) + }) + } + + /// Start a backend from verified hibernation artifacts. + pub fn resume( + &self, + id: Uuid, + request: ResumeSandbox, + ) -> Pin> + Send + '_>> { + Box::pin(async move { + let _operation = self.operation_lock(id).lock_owned().await; + let mut instance = self.get(id)?; + require_quiescent_state(&instance, SandboxState::Hibernated)?; + if instance.backend_ownership != BackendOwnership::Stopped { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is hibernated with unresolved backend ownership" + ))); + } + if self.backend_owner(id).is_some() { + self.mark_instance_recovery(instance)?; + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is hibernated but still retains a backend owner" + ))); + } + + let hibernate_dir = self.hibernate_dir(id); + let manifest = load_and_verify_manifest(&hibernate_dir) + .await + .map_err(|error| { + let recovery = self.mark_instance_recovery(instance.clone()).err(); + with_recovery_error( + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} hibernation artifacts are invalid: {error}" + )), + recovery, + ) + })?; + if let Err(error) = validate_manifest_identity(&manifest, &instance) { + let recovery = self.mark_instance_recovery(instance).err(); + return Err(with_recovery_error(error, recovery)); + } + let spawner = self.spawner(instance.backend).ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} has no resume adapter for {}", + instance.backend + )) + })?; + let capability = spawner + .restore_capability(&request.binary_path) + .await? + .ok_or_else(|| { + BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} backend {} does not support resume", + instance.backend + )) + })?; + if capability.backend != manifest.backend + || capability.version != manifest.backend_version + || capability.snapshot_kind != manifest.snapshot_kind + { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "instance {id} hibernation image is incompatible with the current resume adapter" + ))); + } + let storage = self.storage.reconstruct(&id.to_string()).await?; + + instance.begin_resume_operation()?; + instance.transition(SandboxState::Resuming)?; + crate::failpoint::state("resume-begin-state") + .and_then(|_| self.persist_and_retain(instance.clone()))?; + crate::failpoint::pause("resume-after-begin").await; + + let run_dir = self.instance_dir(id); + if let Err(error) = spawner.prepare_spawn(&run_dir).await { + return Err(self.fail_resume_without_owner( + instance, + format!("resume ownership preparation failed: {error}"), + )); + } + instance.backend_ownership = BackendOwnership::Starting; + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendStarting) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-starting-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_without_owner( + instance, + format!("resume ownership intent commit failed: {error}"), + )); + } + + let restored = match crate::failpoint::backend("resume-backend-start") { + Ok(()) => { + spawner + .restore(RestoreRequest { + instance_id: id, + run_dir, + binary_path: request.binary_path, + storage, + snapshot_path: hibernate_dir.join(VMSTATE_ARTIFACT), + mem_path: hibernate_dir.join(MEMORY_ARTIFACT), + checkpoint_backend: manifest.backend, + expected_version: manifest.backend_version.clone(), + snapshot_kind: manifest.snapshot_kind, + expose_guest_socket: manifest.expose_guest_socket, + }) + .await + } + Err(error) => Err(crate::spawner::SpawnFailure::clean(error)), + }; + let restored = match restored { + Ok(owner) => owner, + Err(error) => { + let (source, owner) = error.into_parts(); + if let Some(owner) = owner { + let _ = self.retain_backend(id, owner); + instance.backend_ownership = BackendOwnership::Running; + return Err(self.fail_resume_with_owner( + instance, + format!("resume backend start failed: {source}"), + )); + } + return Err(self.fail_resume_without_owner( + instance, + format!("resume backend start failed: {source}"), + )); + } + }; + instance.backend_ownership = BackendOwnership::Running; + if let Some(error) = self.retain_backend(id, restored.clone()) { + return Err(self.fail_resume_with_owner(instance, error)); + } + if restored.instance_id() != id + || restored.backend() != manifest.backend + || restored.version().map(str::to_string) != manifest.backend_version + { + return Err(self + .abort_resumed_backend( + instance, + &restored, + "restored backend identity does not match the hibernation manifest" + .to_string(), + ) + .await); + } + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendStarted) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-started-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_with_owner( + instance, + format!("restored backend ownership commit failed: {error}"), + )); + } + if let Err(error) = self + .verify_resumed_backend(id, &restored, manifest.expose_guest_socket) + .await + { + return Err(self + .abort_resumed_backend( + instance, + &restored, + format!("restored backend readiness failed: {error}"), + ) + .await); + } + if let Err(error) = instance + .advance_resume_phase(OperationPhase::ResumeBackendReady) + .map_err(BlazeDaemonError::from) + .and_then(|_| { + crate::failpoint::state("resume-ready-state")?; + self.persist_and_retain(instance.clone()) + }) + { + return Err(self.fail_resume_with_owner( + instance, + format!("restored backend readiness commit failed: {error}"), + )); + } + + let recovery = instance.clone(); + instance.transition(SandboxState::Running)?; + instance.finish_operation(); + if let Err(error) = crate::failpoint::state("resume-final-state") + .and_then(|_| self.persist_and_retain(instance.clone())) + { + return Err(self.fail_resume_with_owner( + recovery, + format!("final running-state commit failed: {error}"), + )); + } + Ok(instance) + }) + } + + async fn compensate_hibernate( + &self, + mut instance: SandboxInstance, + backend: &DynBackendInstance, + staging_dir: Option<&Path>, + cause: String, + ) -> BlazeDaemonError { + let resumed = match crate::failpoint::backend("hibernate-compensation-resume") { + Ok(()) => backend.resume().await, + Err(error) => Err(error), + }; + if let Err(error) = resumed { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; backend resume compensation failed: {error}"), + ); + } + if let Err(error) = self + .verify_resumed_backend( + instance.id, + backend, + !backend.guest_socket_path().as_os_str().is_empty(), + ) + .await + { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; resumed backend readiness failed: {error}"), + ); + } + if let Some(staging_dir) = staging_dir + && let Err(error) = remove_directory_and_sync(staging_dir).await + { + return self.fail_hibernate_after_stop( + instance, + format!("{cause}; staging cleanup failed: {error}"), + ); + } + let recovery = instance.clone(); + instance.backend_ownership = BackendOwnership::Running; + if let Err(error) = instance.transition(SandboxState::Running) { + return self.fail_hibernate_after_stop( + recovery, + format!("{cause}; running-state compensation failed: {error}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + return self.fail_hibernate_after_stop( + recovery, + format!("{cause}; running-state compensation commit failed: {error}"), + ); + } + BlazeDaemonError::Internal(cause) + } + + async fn verify_resumed_backend( + &self, + id: Uuid, + backend: &DynBackendInstance, + expose_guest_socket: bool, + ) -> Result<()> { + require_backend_live(id, backend).await?; + if expose_guest_socket { + self.wait_for_guest_ready(backend, "resume-guest-ready") + .await?; + } + require_backend_live(id, backend).await + } + + async fn abort_resumed_backend( + &self, + mut instance: SandboxInstance, + backend: &DynBackendInstance, + cause: String, + ) -> BlazeDaemonError { + let stopped = match crate::failpoint::backend("resume-backend-stop") { + Ok(()) => backend.kill().await, + Err(error) => Err(error), + }; + if let Err(error) = stopped { + instance.backend_ownership = BackendOwnership::Unknown; + return self.fail_resume_with_owner( + instance, + format!("{cause}; restored backend termination failed: {error}"), + ); + } + self.remove_backend_owner(instance.id); + instance.backend_ownership = BackendOwnership::Stopped; + self.fail_resume_without_owner(instance, cause) + } + + fn fail_resume_without_owner( + &self, + mut instance: SandboxInstance, + cause: String, + ) -> BlazeDaemonError { + let recovery = instance.clone(); + instance.backend_ownership = BackendOwnership::Stopped; + if let Err(error) = instance.transition(SandboxState::Hibernated) { + return self.fail_resume_with_owner( + recovery, + format!("{cause}; hibernated-state compensation failed: {error}"), + ); + } + instance.finish_operation(); + if let Err(error) = self.persist_and_retain(instance) { + return self.fail_resume_with_owner( + recovery, + format!("{cause}; hibernated-state compensation commit failed: {error}"), + ); + } + BlazeDaemonError::Internal(cause) + } + + fn fail_hibernate_after_stop( + &self, + instance: SandboxInstance, + cause: String, + ) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "hibernate {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } + + fn fail_resume_with_owner(&self, instance: SandboxInstance, cause: String) -> BlazeDaemonError { + let id = instance.id; + let recovery = self.mark_instance_recovery(instance).err(); + BlazeDaemonError::RecoveryRequired(format!( + "resume {id}: {cause}; resources retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + )) + } + + pub(super) async fn cleanup_hibernate_artifacts(&self, id: Uuid) -> Result<()> { + let instance_dir = self.instance_dir(id); + let mut entries = match tokio::fs::read_dir(&instance_dir).await { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error.into()), + }; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name == HIBERNATE_DIRECTORY + || (name.starts_with(".hibernate.") + && (name.ends_with(".tmp") || name.ends_with(".bak"))) + { + remove_directory_and_sync(&entry.path()).await?; + } + } + Ok(()) + } + + fn instance_dir(&self, id: Uuid) -> PathBuf { + self.state_dir.join(id.to_string()) + } + + fn hibernate_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id).join(HIBERNATE_DIRECTORY) + } + + fn hibernate_staging_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id) + .join(format!(".hibernate.{}.tmp", Uuid::new_v4())) + } + + fn hibernate_backup_dir(&self, id: Uuid) -> PathBuf { + self.instance_dir(id) + .join(format!(".hibernate.{}.bak", Uuid::new_v4())) + } +} + +fn require_quiescent_state(instance: &SandboxInstance, expected: SandboxState) -> Result<()> { + if let Some(journal) = &instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {} has unfinished {} operation", + instance.id, journal.kind + ))); + } + if instance.state != expected { + return Err(BlazeDaemonError::Conflict(format!( + "instance {} is {}, expected {expected}", + instance.id, instance.state + ))); + } + Ok(()) +} + +async fn require_backend_live(id: Uuid, backend: &DynBackendInstance) -> Result<()> { + match backend.try_wait().await { + Ok(None) => Ok(()), + Ok(Some(result)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend exited (exit={:?}, signal={:?})", + result.exit_code, result.signal + ))), + Err(error) => Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend liveness is unknown: {error}" + ))), + } +} + +async fn build_hibernate_manifest( + directory: &Path, + instance: &SandboxInstance, + backend_version: Option, + expose_guest_socket: bool, +) -> Result { + let mut artifacts = Vec::with_capacity(REQUIRED_ARTIFACTS.len()); + for name in REQUIRED_ARTIFACTS { + artifacts.push(hash_artifact(&directory.join(name), name).await?); + } + Ok(HibernateManifest { + format_version: HIBERNATE_FORMAT_VERSION, + sandbox_id: instance.id, + policy_name: instance.policy_name.clone(), + image_digest: instance.image_digest.clone(), + backend: instance.backend, + backend_version, + snapshot_kind: SnapshotKind::Full, + expose_guest_socket, + artifacts, + }) +} + +async fn write_and_sync_manifest(directory: &Path, manifest: &HibernateManifest) -> Result<()> { + for name in REQUIRED_ARTIFACTS { + tokio::fs::File::open(directory.join(name)) + .await? + .sync_all() + .await?; + } + let mut encoded = serde_json::to_vec_pretty(manifest)?; + encoded.push(b'\n'); + let mut file = tokio::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(directory.join(MANIFEST_ARTIFACT)) + .await?; + file.write_all(&encoded).await?; + file.sync_all().await?; + drop(file); + sync_directory(directory.to_path_buf()).await +} + +async fn load_and_verify_manifest(directory: &Path) -> Result { + let manifest_path = directory.join(MANIFEST_ARTIFACT); + let manifest_metadata = tokio::fs::symlink_metadata(&manifest_path).await?; + if manifest_metadata.file_type().is_symlink() || !manifest_metadata.is_file() { + return Err(BlazeDaemonError::Internal(format!( + "hibernate manifest {} is not a regular file", + manifest_path.display() + ))); + } + let manifest: HibernateManifest = + serde_json::from_slice(&tokio::fs::read(&manifest_path).await?)?; + if manifest.format_version != HIBERNATE_FORMAT_VERSION { + return Err(BlazeDaemonError::UnsupportedOperation(format!( + "unsupported hibernation format {}", + manifest.format_version + ))); + } + if manifest.snapshot_kind != SnapshotKind::Full { + return Err(BlazeDaemonError::UnsupportedOperation( + "hibernation image is not self-contained".to_string(), + )); + } + let expected_names = REQUIRED_ARTIFACTS + .into_iter() + .map(str::to_string) + .collect::>(); + let observed_names = manifest + .artifacts + .iter() + .map(|artifact| artifact.name.clone()) + .collect::>(); + if observed_names != expected_names || manifest.artifacts.len() != REQUIRED_ARTIFACTS.len() { + return Err(BlazeDaemonError::Internal( + "hibernation manifest has an invalid artifact set".to_string(), + )); + } + let directory_names = read_directory_names(directory).await?; + let expected_directory_names = [ + MANIFEST_ARTIFACT.to_string(), + MEMORY_ARTIFACT.to_string(), + VMSTATE_ARTIFACT.to_string(), + ] + .into_iter() + .collect::>(); + if directory_names != expected_directory_names { + return Err(BlazeDaemonError::Internal( + "hibernation directory has an unexpected file set".to_string(), + )); + } + for artifact in &manifest.artifacts { + let path = directory.join(&artifact.name); + let metadata = tokio::fs::symlink_metadata(&path).await?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(BlazeDaemonError::Internal(format!( + "hibernate artifact {} is not a regular file", + path.display() + ))); + } + let observed = hash_artifact(&path, &artifact.name).await?; + if &observed != artifact { + return Err(BlazeDaemonError::Internal(format!( + "hibernate artifact {} failed integrity verification", + artifact.name + ))); + } + } + Ok(manifest) +} + +fn validate_manifest_identity( + manifest: &HibernateManifest, + instance: &SandboxInstance, +) -> Result<()> { + if manifest.sandbox_id != instance.id + || manifest.policy_name != instance.policy_name + || manifest.image_digest != instance.image_digest + || manifest.backend != instance.backend + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {} hibernation identity does not match durable lifecycle state", + instance.id + ))); + } + Ok(()) +} + +async fn hash_artifact(path: &Path, name: &str) -> Result { + let mut file = tokio::fs::File::open(path).await?; + let mut hasher = Sha256::new(); + let mut size_bytes = 0_u64; + let mut buffer = [0_u8; 128 * 1024]; + loop { + let read = file.read(&mut buffer).await?; + if read == 0 { + break; + } + size_bytes = size_bytes + .checked_add(read as u64) + .ok_or_else(|| BlazeDaemonError::Internal("artifact size overflow".to_string()))?; + hasher.update(&buffer[..read]); + } + Ok(CheckpointArtifact { + name: name.to_string(), + size_bytes, + sha256: format!("{:x}", hasher.finalize()), + }) +} + +async fn read_directory_names(directory: &Path) -> Result> { + let mut names = BTreeSet::new(); + let mut entries = tokio::fs::read_dir(directory).await?; + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name().into_string().map_err(|_| { + BlazeDaemonError::Internal(format!( + "hibernate directory {} contains a non-UTF-8 name", + directory.display() + )) + })?; + names.insert(name); + } + Ok(names) +} + +async fn prepare_hibernate_directory(final_dir: &Path) -> Result<()> { + let parent = final_dir.parent().ok_or_else(|| { + BlazeDaemonError::Internal(format!( + "hibernate directory {} has no parent", + final_dir.display() + )) + })?; + let mut entries = tokio::fs::read_dir(parent).await?; + let mut obsolete_backups = Vec::new(); + while let Some(entry) = entries.next_entry().await? { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(".hibernate.") && name.ends_with(".tmp") { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance directory {} contains unfinished hibernation artifacts", + parent.display() + ))); + } + if name.starts_with(".hibernate.") && name.ends_with(".bak") { + if !final_dir.is_dir() { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance directory {} contains an unpaired hibernation backup", + parent.display() + ))); + } + obsolete_backups.push(entry.path()); + } + } + for backup in obsolete_backups { + remove_directory_and_sync(&backup).await?; + } + Ok(()) +} + +async fn sync_directory(directory: PathBuf) -> Result<()> { + tokio::fs::File::open(directory).await?.sync_all().await?; + Ok(()) +} + +async fn remove_directory_and_sync(directory: &Path) -> Result<()> { + match tokio::fs::remove_dir_all(directory).await { + Ok(()) => { + let parent = directory.parent().ok_or_else(|| { + BlazeDaemonError::Internal(format!( + "removed hibernation path {} has no parent", + directory.display() + )) + })?; + sync_directory(parent.to_path_buf()).await + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(error.into()), + } +} + +fn with_recovery_error( + error: BlazeDaemonError, + recovery: Option, +) -> BlazeDaemonError { + match recovery { + Some(recovery) => BlazeDaemonError::RecoveryRequired(format!( + "{error}; recovery state persistence failed: {recovery}" + )), + None => error, + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 7891a818d8..3bad0562a5 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -82,7 +82,7 @@ pub struct SandboxManager { spawners: Arc, active_backend: BackendKind, pub(super) storage: Arc, - state_dir: PathBuf, + pub(super) state_dir: PathBuf, pub(super) checkpoints: CheckpointStore, rootfs_size: u64, mem_size: u64, @@ -251,7 +251,7 @@ impl SandboxManager { .lock() .map_err(|_| poisoned("instances"))? .values() - .filter(|instance| !is_clean_terminal(instance)) + .filter(|instance| requires_automatic_cleanup(instance)) .map(|instance| instance.id) .collect::>(); ids.extend( @@ -879,6 +879,17 @@ impl SandboxManager { ))); } + if let Err(error) = self.cleanup_hibernate_artifacts(id).await { + let recovery = self.mark_recovery(id).err(); + return Err(BlazeDaemonError::RecoveryRequired(format!( + "destroy {id}: backend stopped but hibernation cleanup failed: {error}; \ + storage retained{}", + recovery + .map(|error| format!("; recovery state persistence failed: {error}")) + .unwrap_or_default() + ))); + } + if let Err(error) = self.storage.release_by_id(&id.to_string()).await { let recovery = self.mark_recovery(id).err(); return Err(BlazeDaemonError::RecoveryRequired(format!( @@ -928,7 +939,10 @@ impl SandboxManager { /// Reconcile every record that is not cleanly terminal without aborting /// on one failure. pub async fn reconcile_startup(&self) -> ReconcileReport { - self.cleanup_owned_instances().await + let mut classification_failures = self.classify_interrupted_hibernation(); + let mut report = self.cleanup_owned_instances().await; + report.failures.append(&mut classification_failures); + report } /// Cancel readiness polling before the daemon drains active requests. @@ -943,6 +957,50 @@ impl SandboxManager { .await } + fn classify_interrupted_hibernation(&self) -> Vec { + let interrupted = match self.instances.lock() { + Ok(instances) => instances + .values() + .filter(|instance| { + matches!( + instance.state, + SandboxState::Hibernating | SandboxState::Resuming + ) || matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ) + }) + .cloned() + .collect::>(), + Err(poisoned) => poisoned + .into_inner() + .values() + .filter(|instance| { + matches!( + instance.state, + SandboxState::Hibernating | SandboxState::Resuming + ) || matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ) + }) + .cloned() + .collect::>(), + }; + interrupted + .into_iter() + .filter_map(|instance| { + let id = instance.id; + self.mark_instance_recovery(instance) + .err() + .map(|error| ReconcileFailure { + instance_id: id, + error: format!("interrupted hibernation classification failed: {error}"), + }) + }) + .collect() + } + fn guest_client(&self, id: Uuid) -> Result { let backend = self .backend_instances @@ -993,13 +1051,13 @@ impl SandboxManager { let mut ids = match self.instances.lock() { Ok(instances) => instances .values() - .filter(|instance| !is_clean_terminal(instance)) + .filter(|instance| requires_automatic_cleanup(instance)) .map(|instance| instance.id) .collect::>(), Err(poisoned) => poisoned .into_inner() .values() - .filter(|instance| !is_clean_terminal(instance)) + .filter(|instance| requires_automatic_cleanup(instance)) .map(|instance| instance.id) .collect::>(), }; @@ -1302,6 +1360,18 @@ fn is_clean_terminal(instance: &SandboxInstance) -> bool { ) } +fn requires_automatic_cleanup(instance: &SandboxInstance) -> bool { + !(is_clean_terminal(instance) + || (instance.state == SandboxState::Hibernated + && instance.operation.is_none() + && instance.backend_ownership == BackendOwnership::Stopped) + || (instance.state == SandboxState::RecoveryRequired + && matches!( + instance.operation.as_ref().map(|operation| operation.kind), + Some(OperationKind::Hibernate | OperationKind::Resume) + ))) +} + fn operation_lock( locks: &mut HashMap>>, id: Uuid, From 5539bd8f099519394354b286886a5c93e1ae6e17 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 13:59:51 +0800 Subject: [PATCH 34/40] docs(blaze): explain sandbox hibernation This explains when hibernation is available, which API routes release and resume a sandbox backend, and which durable checks users can expect before either state change becomes visible. The English and Chinese guides describe artifact verification, restart behavior, failure ownership, explicit cleanup, and the disk-space tradeoff of retaining the latest hibernation image. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 43 +++++++++++++++++++++++++++++++++++++----- src/blaze/README_zh.md | 33 +++++++++++++++++++++++++++++--- 2 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/blaze/README.md b/src/blaze/README.md index 6baed6c9ad..f3ccacf3b1 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -13,8 +13,9 @@ Designed as the per-host agent for E2B-style orchestrator platforms. - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **Policy-driven backend selection** — workload class → backend priority list -- **Lifecycle state machine** — 9 states: Pending, Creating, Running, Paused, - Checkpointed, RecoveryRequired, Reset, Warm, and Destroyed +- **Lifecycle state machine** — 13 states: Pending, Creating, Running, Paused, + Checkpointed, Restoring, Hibernating, Hibernated, Resuming, + RecoveryRequired, Reset, Warm, and Destroyed - **Guest operations** — bounded command execution and file transfer for running backends that expose a guest endpoint - **Warm pool management** — pre-warmed instances with TTL-based GC @@ -141,6 +142,8 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/sandboxes/{id}/checkpoint` | Capture a full checkpoint when the backend and storage provider support it | | GET | `/v1/sandboxes/{id}/checkpoints` | List committed checkpoints and HEAD reachability | | POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | Replace a running sandbox from a verified checkpoint | +| POST | `/v1/sandboxes/{id}/hibernate` | Persist VM state and release the live backend | +| POST | `/v1/sandboxes/{id}/resume` | Resume a hibernated sandbox and wait for enabled guest transport | | GET | `/v1/instances` | Alias for listing sandboxes | | POST | `/v1/instances` | Alias for creating a sandbox | | GET | `/v1/instances/{id}` | Alias for sandbox details | @@ -152,6 +155,8 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | POST | `/v1/instances/{id}/checkpoint` | Compatible full-checkpoint action | | GET | `/v1/instances/{id}/checkpoints` | Compatible checkpoint-list action | | POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | Compatible checkpoint-restore action | +| POST | `/v1/instances/{id}/hibernate` | Compatible sandbox-hibernation action | +| POST | `/v1/instances/{id}/resume` | Compatible sandbox-resume action | | POST | `/v1/instances/{id}/reset` | Reserved; returns `501` until runtime reset is implemented | | GET | `/v1/pools` | List warm pools | | GET | `/v1/pools/{backend}/{class}` | Get pool status | @@ -172,9 +177,12 @@ resources. A successful create finishes in `Running`; a successful destroy finishes in `Destroyed`. If compensation cannot release every owned resource, the sandbox remains visible as `RecoveryRequired` so destroy can be retried. -At startup, the daemon reconciles each non-terminal sandbox independently. -Failure to clean up one sandbox does not prevent the remaining records from -being processed or the API from starting. +At startup, the daemon reconciles each sandbox that requires automatic cleanup +independently. A completed hibernation is retained for resume. An interrupted +hibernate or resume is retained as `RecoveryRequired` for explicit destroy +instead of being mistaken for a live runtime. Failure to clean up one of the +remaining sandboxes does not prevent the other records from being processed or +the API from starting. During graceful shutdown, the daemon first stops accepting work and drains accepted connections. It then attempts bounded cleanup for every persisted @@ -229,6 +237,31 @@ later destroy can finish cleanup without losing process ownership. `last_checkpoint` continues to mean the most recent completed capture. Restore moves catalog HEAD but does not rewrite capture history. +Hibernation is available only when the running backend supports pause and full +snapshot capture and its configured adapter can restore the same backend +version. These checks happen before the lifecycle journal changes. A successful +hibernate: + +1. records intent, pauses the backend, and writes VM state and memory into a + hidden staging directory; +2. flushes the retained storage slot and records artifact sizes and SHA-256 + digests in a manifest; +3. synchronizes the complete image before stopping the backend; +4. publishes the hibernation directory and commits `Hibernated`. + +Resume verifies the manifest identity, exact file set, and artifact digests +before starting a replacement backend. The manager owns that backend before +waiting for optional guest readiness and commits `Running` only after a final +liveness check. A failure before the original backend stops resumes it and +rechecks enabled guest transport. A clean resume failure returns to +`Hibernated`; if cleanup cannot be confirmed, the replacement owner and +operation journal remain available through `RecoveryRequired`. + +The storage slot remains allocated while hibernated. A successful resume also +retains the latest hibernation image until the next hibernate replaces it or an +explicit destroy removes it. The daemon does not automatically complete an +interrupted hibernate or resume after restart. + Runtime reset remains reserved and returns `501` without changing runtime or persisted state. diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index d8b5662b90..77f33b762e 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -12,8 +12,9 @@ Prometheus 指标导出，设计为 E2B 类编排平台的单机执行代理。 - **HTTP API** — Unix domain socket (`/run/blaze/api.sock`) + TCP (`:14159`) - **策略驱动后端选择** — workload class → 后端优先级列表 -- **生命周期状态机** — 9 种状态：Pending、Creating、Running、Paused、 - Checkpointed、RecoveryRequired、Reset、Warm 和 Destroyed +- **生命周期状态机** — 13 种状态：Pending、Creating、Running、Paused、 + Checkpointed、Restoring、Hibernating、Hibernated、Resuming、 + RecoveryRequired、Reset、Warm 和 Destroyed - **Guest 操作** — 对提供 guest endpoint 的运行中后端执行有界命令和文件传输 - **Warm pool 管理** — 预热实例 + 基于 TTL 的 GC - **模板注册表** — 内存中模板追踪，支持空闲驱逐 @@ -137,6 +138,8 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/sandboxes/{id}/checkpoint` | 后端和存储 provider 支持时捕获完整 checkpoint | | GET | `/v1/sandboxes/{id}/checkpoints` | 列出已提交的 checkpoint 及其 HEAD 可达性 | | POST | `/v1/sandboxes/{id}/rollback/{checkpoint_id}` | 用经过校验的 checkpoint 替换正在运行的 sandbox | +| POST | `/v1/sandboxes/{id}/hibernate` | 持久化 VM 状态并释放正在运行的后端 | +| POST | `/v1/sandboxes/{id}/resume` | 恢复休眠 sandbox，并等待已启用的 guest 通信就绪 | | GET | `/v1/instances` | 列出 sandbox 的兼容入口 | | POST | `/v1/instances` | 创建 sandbox 的兼容入口 | | GET | `/v1/instances/{id}` | 获取 sandbox 详情的兼容入口 | @@ -148,6 +151,8 @@ images_dir = "/var/lib/blaze/images" | POST | `/v1/instances/{id}/checkpoint` | 捕获完整 checkpoint 的兼容入口 | | GET | `/v1/instances/{id}/checkpoints` | 列出 checkpoint 的兼容入口 | | POST | `/v1/instances/{id}/rollback/{checkpoint_id}` | 恢复 checkpoint 的兼容入口 | +| POST | `/v1/instances/{id}/hibernate` | 休眠 sandbox 的兼容入口 | +| POST | `/v1/instances/{id}/resume` | 恢复休眠 sandbox 的兼容入口 | | POST | `/v1/instances/{id}/reset` | 预留接口；运行时重置实现前返回 `501` | | GET | `/v1/pools` | 列出 warm pool | | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | @@ -167,7 +172,9 @@ images_dir = "/var/lib/blaze/images" `Running`，销毁成功后状态为 `Destroyed`。如果失败补偿不能释放全部已有 资源，sandbox 会保留为可查询的 `RecoveryRequired`，后续可以再次执行销毁。 -daemon 启动时会逐个处理未结束的 sandbox。单个 sandbox 清理失败不会阻止 +daemon 启动时会逐个处理需要自动清理的 sandbox。已经完成休眠的 sandbox 会 +保留下来等待恢复；中断的休眠或恢复操作会以 `RecoveryRequired` 保留，等待 +显式销毁，而不会被误认为仍在运行。其余 sandbox 中的单条清理失败不会阻止 其他记录继续处理，也不会阻止 API 启动。 正常关闭时，daemon 会先停止接收新请求并等待已有连接结束，再为每条持久化 @@ -212,6 +219,26 @@ sandbox 标记为 `RecoveryRequired`，后续 destroy 仍能找到并清理这 `last_checkpoint` 始终表示最近一次成功捕获。恢复只移动 catalog HEAD，不会 改写捕获历史。 +只有运行中后端支持暂停和完整快照，并且配置的 adapter 能恢复相同的后端版本 +时，休眠才可用。这些检查会在生命周期操作记录发生变化前完成。一次成功的 +休眠会： + +1. 记录操作意图、暂停后端，并把 VM 状态和内存写入隐藏的暂存目录； +2. 刷新保留的存储 slot，并在 manifest 中记录文件大小和 SHA-256 摘要； +3. 在停止后端之前同步完整的休眠镜像； +4. 发布休眠目录，并提交 `Hibernated` 状态。 + +恢复会在启动替换后端前校验 manifest 身份、完整文件集合和文件摘要。manager +会先取得新后端的归属，再等待可选的 guest 通信就绪；只有最后一次存活检查 +通过后才提交 `Running`。旧后端停止前发生失败时，daemon 会恢复旧后端，并 +重新检查已经启用的 guest 通信。可以完整清理的恢复失败会回到 +`Hibernated`；如果无法确认清理结果，替换后端归属和操作记录会通过 +`RecoveryRequired` 保留下来。 + +休眠期间，存储 slot 会继续保留。恢复成功后，最近一次休眠镜像也会保留到 +下一次休眠替换它，或显式销毁将其删除。daemon 重启后不会自动完成中断的 +休眠或恢复操作。 + runtime reset 仍是预留接口，会返回 `501`，且不会修改 runtime 或持久化状态。 ### Guest 操作 From 2591e13ee2edfb8329ecf7a79033a9ca3540e7a5 Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 15:15:11 +0800 Subject: [PATCH 35/40] feat(blaze): synchronize running sandbox storage This enables operators to schedule bounded provider synchronization for running sandboxes while leaving the default schedule disabled. Each sweep enters the sandbox operation lock, rechecks lifecycle and journal state, confirms backend and storage ownership, and isolates one provider failure from later sandboxes. Attempt deadlines retain the slot for retry. The daemon supervises the worker, stops and joins it before connection drain and runtime cleanup, and preserves both worker and shutdown failures. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/config.rs | 105 ++- src/blaze/crates/blaze-core/src/storage.rs | 4 + src/blaze/crates/blazed/src/daemon.rs | 73 +- src/blaze/crates/blazed/src/sandbox.rs | 2 + src/blaze/crates/blazed/src/sandbox/flush.rs | 730 ++++++++++++++++++ .../crates/blazed/src/sandbox/manager.rs | 14 +- src/blaze/crates/blazed/src/state.rs | 4 +- 7 files changed, 921 insertions(+), 11 deletions(-) create mode 100644 src/blaze/crates/blazed/src/sandbox/flush.rs diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index 2c490167a2..0695d62952 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -4,10 +4,12 @@ use std::collections::HashMap; use std::fs; use std::path::{Path, PathBuf}; +use std::time::Duration; use serde::{Deserialize, Serialize}; use crate::error::{BlazeError, ConfigErrorSource, Result}; +use crate::policy::parse_duration; /// Top-level daemon configuration. #[derive(Debug, Clone, Serialize, Deserialize, Default)] @@ -178,11 +180,16 @@ pub struct StorageSection { #[serde(default)] pub prefork: bool, - /// Interval for flushing dirty data. - /// NOTE: Reserved for future use. Not yet wired into runtime. + /// Interval for synchronizing provider-owned runtime data. + /// + /// The literal `disabled` turns off periodic synchronization. #[serde(default = "default_flush_interval")] pub flush_interval: String, + /// Maximum duration of one provider synchronization attempt. + #[serde(default = "default_flush_timeout")] + pub flush_timeout: String, + /// Logical size of file-provider root filesystem slots. #[serde(default = "default_rootfs_size")] pub rootfs_size: u64, @@ -201,12 +208,40 @@ impl Default for StorageSection { pool_size: 0, prefork: false, flush_interval: default_flush_interval(), + flush_timeout: default_flush_timeout(), rootfs_size: default_rootfs_size(), mem_size: default_mem_size(), } } } +/// Parsed periodic storage synchronization policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StorageFlushSchedule { + /// Do not run periodic synchronization. + Disabled, + /// Run one sweep after every configured interval. + Every(Duration), +} + +impl StorageSection { + /// Parse the periodic synchronization setting. + pub fn flush_schedule(&self) -> Result { + if self.flush_interval == "disabled" { + return Ok(StorageFlushSchedule::Disabled); + } + parse_duration(&self.flush_interval) + .map(StorageFlushSchedule::Every) + .ok_or_else(|| invalid_storage_duration("flush_interval", &self.flush_interval, true)) + } + + /// Parse the maximum duration of one provider synchronization attempt. + pub fn flush_timeout_duration(&self) -> Result { + parse_duration(&self.flush_timeout) + .ok_or_else(|| invalid_storage_duration("flush_timeout", &self.flush_timeout, false)) + } +} + impl DaemonConfig { /// Load and parse a daemon configuration file at `path`. pub fn load(path: &Path) -> Result { @@ -226,7 +261,23 @@ impl DaemonConfig { ), }); } - validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir) + validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir)?; + self.storage.flush_schedule()?; + self.storage.flush_timeout_duration()?; + Ok(()) + } +} + +fn invalid_storage_duration(name: &str, value: &str, allow_disabled: bool) -> BlazeError { + let expected = if allow_disabled { + "a positive duration or \"disabled\"" + } else { + "a positive duration" + }; + BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "storage.{name} ({value:?}) must be {expected}" + )), } } @@ -295,6 +346,9 @@ fn default_storage_provider() -> String { "file".to_string() } fn default_flush_interval() -> String { + "disabled".to_string() +} +fn default_flush_timeout() -> String { "30s".to_string() } fn default_rootfs_size() -> u64 { @@ -316,6 +370,10 @@ mod tests { assert_eq!(cfg.api.max_body_bytes, 1024 * 1024); assert!(cfg.backends.is_empty()); assert_ne!(cfg.storage.images_dir, cfg.storage.instances_dir); + assert_eq!( + cfg.storage.flush_schedule().expect("flush schedule"), + StorageFlushSchedule::Disabled + ); } #[test] @@ -384,4 +442,45 @@ mod tests { assert!(error.to_string().contains("must be disjoint")); } } + + #[test] + fn storage_flush_schedule_accepts_disabled_or_positive_duration() { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_interval = "disabled".into(); + cfg.validate().expect("disabled schedule"); + assert_eq!( + cfg.storage.flush_schedule().expect("schedule"), + StorageFlushSchedule::Disabled + ); + + cfg.storage.flush_interval = "15s".into(); + cfg.validate().expect("positive schedule"); + assert_eq!( + cfg.storage.flush_schedule().expect("schedule"), + StorageFlushSchedule::Every(Duration::from_secs(15)) + ); + } + + #[test] + fn storage_flush_schedule_rejects_invalid_values() { + for interval in ["0s", "not-a-duration"] { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_interval = interval.into(); + let error = cfg.validate().expect_err("invalid flush interval"); + assert!( + error.to_string().contains("storage.flush_interval"), + "{error}" + ); + } + + for timeout in ["0s", "disabled", "not-a-duration"] { + let mut cfg = DaemonConfig::default(); + cfg.storage.flush_timeout = timeout.into(); + let error = cfg.validate().expect_err("invalid flush timeout"); + assert!( + error.to_string().contains("storage.flush_timeout"), + "{error}" + ); + } + } } diff --git a/src/blaze/crates/blaze-core/src/storage.rs b/src/blaze/crates/blaze-core/src/storage.rs index aa4961a6ca..88777de464 100644 --- a/src/blaze/crates/blaze-core/src/storage.rs +++ b/src/blaze/crates/blaze-core/src/storage.rs @@ -126,6 +126,10 @@ pub trait StorageProvider: Send + Sync { async fn reconstruct(&self, instance_id: &str) -> Result; /// Flush dirty data to persistent storage (implementation may be no-op). + /// + /// The daemon may cancel this future when its configured attempt deadline + /// or shutdown signal wins. Cancellation must retain slot ownership and + /// leave a later synchronization or cleanup attempt safe. async fn flush_dirty(&self, slot: &StorageSlot) -> Result<()>; /// Query warm pool status. diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 2a30ddf85f..8cb2762d1e 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use std::time::Duration; use blaze_core::backend::BackendKind; -use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode}; +use blaze_core::config::{DaemonConfig, PolicyLoadErrorMode, StorageFlushSchedule}; use blaze_core::kernel::HookRegistry; use blaze_core::policy::PolicyEngine; use blaze_core::pool::PoolManager; @@ -24,6 +24,7 @@ use tokio::task::{JoinError, JoinSet}; use crate::api; use crate::daemon_socket::{DaemonLock, DaemonSocket}; use crate::error::{BlazeDaemonError, Result}; +use crate::sandbox::FlushLoop; use crate::spawner::{ BubblewrapSpawner, DynSpawner, FirecrackerSpawner, MockSpawner, SpawnerRegistry, }; @@ -142,6 +143,8 @@ fn report_connection_result(result: std::result::Result<(), JoinError>) -> bool /// bind the API socket, and run the accept loop until SIGTERM/SIGINT. pub async fn run(config_path: &Path) -> Result<()> { let config = DaemonConfig::load(config_path)?; + let flush_schedule = config.storage.flush_schedule()?; + let flush_timeout = config.storage.flush_timeout_duration()?; tracing::info!(?config_path, "loaded daemon config"); ensure_dirs(&config)?; @@ -220,7 +223,7 @@ pub async fn run(config_path: &Path) -> Result<()> { None }; - serve(listener, tcp_listener, state).await + serve(listener, tcp_listener, state, flush_schedule, flush_timeout).await } fn ensure_dirs(cfg: &DaemonConfig) -> Result<()> { @@ -341,6 +344,8 @@ async fn serve( mut uds: DaemonSocket, tcp: Option, state: Arc, + flush_schedule: StorageFlushSchedule, + flush_timeout: Duration, ) -> Result<()> { let mut sighup = signal(SignalKind::hangup()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGHUP handler: {e}")))?; @@ -349,9 +354,23 @@ async fn serve( let mut sigint = signal(SignalKind::interrupt()) .map_err(|e| BlazeDaemonError::Internal(format!("install SIGINT handler: {e}")))?; let mut connections = ConnectionSupervisor::new(); + let mut flush_loop = match flush_schedule { + StorageFlushSchedule::Disabled => { + tracing::info!("periodic provider synchronization is disabled"); + None + } + StorageFlushSchedule::Every(interval) => { + Some(state.manager.start_flush_loop(interval, flush_timeout)) + } + }; + let mut service_result = Ok(()); loop { tokio::select! { + result = observe_flush_exit(&mut flush_loop), if flush_loop.is_some() => { + service_result = result; + break; + } completed = connections.join_next(), if !connections.is_empty() => { if let Some(completed) = completed { report_connection_result(completed); @@ -397,11 +416,40 @@ async fn serve( uds.stop_accepting(); drop(tcp); + if let Some(flush_loop) = flush_loop.as_mut() { + service_result = merge_stage_result( + service_result, + "provider synchronization shutdown", + flush_loop.shutdown().await, + ); + } let drain = connections.shutdown(SHUTDOWN_BUDGET.connection_drain); let cleanup = api::shutdown_instances(&state, SHUTDOWN_BUDGET.runtime_cleanup); - let shutdown_result = finish_shutdown(drain, cleanup).await; + service_result = merge_stage_result( + service_result, + "connection drain and runtime cleanup", + finish_shutdown(drain, cleanup).await, + ); tracing::info!("blaze daemon stopped"); - shutdown_result + service_result +} + +async fn observe_flush_exit(flush_loop: &mut Option) -> Result<()> { + flush_loop + .as_mut() + .expect("flush loop exists while its select branch is enabled") + .observe_exit() + .await +} + +fn merge_stage_result(current: Result<()>, stage: &str, next: Result<()>) -> Result<()> { + match (current, next) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(previous), Err(next)) => Err(BlazeDaemonError::RecoveryRequired(format!( + "{previous}; {stage} also failed: {next}" + ))), + } } async fn finish_shutdown(drain: Drain, cleanup: Cleanup) -> Result<()> @@ -597,6 +645,23 @@ mod tests { assert!(cleanup_started.load(Ordering::Acquire)); } + #[test] + fn service_and_shutdown_failures_are_both_retained() { + let service = Err(BlazeDaemonError::Internal( + "synchronization worker failed".to_string(), + )); + let shutdown = Err(BlazeDaemonError::Internal( + "runtime cleanup failed".to_string(), + )); + + let error = merge_stage_result(service, "coordinated shutdown", shutdown) + .expect_err("both failures must be reported"); + + assert!(error.to_string().contains("synchronization worker failed")); + assert!(error.to_string().contains("coordinated shutdown")); + assert!(error.to_string().contains("runtime cleanup failed")); + } + #[test] fn service_stop_timeout_covers_shutdown_stages() { let unit = include_str!("../../../dist/blazed.service"); diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs index 2952f6f3ab..2faf639587 100644 --- a/src/blaze/crates/blazed/src/sandbox.rs +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 //! Managed sandbox lifecycle and runtime ownership. +mod flush; mod manager; +pub(crate) use flush::FlushLoop; pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; diff --git a/src/blaze/crates/blazed/src/sandbox/flush.rs b/src/blaze/crates/blazed/src/sandbox/flush.rs new file mode 100644 index 0000000000..12ca42e78b --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/flush.rs @@ -0,0 +1,730 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Periodic synchronization of provider-owned sandbox storage. + +use std::sync::Arc; +use std::time::Duration; + +use blaze_core::lifecycle::{BackendOwnership, SandboxState}; +use tokio::task::JoinHandle; +use tokio::time::{Instant, MissedTickBehavior}; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +#[cfg(test)] +use tokio::sync::Notify; + +use crate::error::{BlazeDaemonError, Result}; + +use super::manager::SandboxManager; + +const FLUSH_LOOP_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + +/// Supervised periodic synchronization task. +pub(crate) struct FlushLoop { + cancellation: CancellationToken, + task: Option>, + #[cfg(test)] + started: Arc, +} + +impl FlushLoop { + /// Wait for an early worker exit and report it as a daemon failure. + pub(crate) async fn observe_exit(&mut self) -> Result<()> { + self.join().await?; + Err(BlazeDaemonError::Internal( + "provider synchronization task exited unexpectedly".to_string(), + )) + } + + /// Request cooperative shutdown and join the worker before returning. + pub(crate) async fn shutdown(&mut self) -> Result<()> { + self.cancellation.cancel(); + let Some(task) = self.task.as_mut() else { + return Ok(()); + }; + match tokio::time::timeout(FLUSH_LOOP_SHUTDOWN_TIMEOUT, task).await { + Ok(result) => { + self.task.take(); + result.map_err(join_error) + } + Err(_) => { + let task = self.task.as_mut().expect("flush task is present"); + task.abort(); + let result = task.await; + self.task.take(); + match result { + Err(error) if error.is_cancelled() => Err(BlazeDaemonError::Internal( + "provider synchronization task exceeded its shutdown deadline".to_string(), + )), + Err(error) => Err(join_error(error)), + Ok(()) => Err(BlazeDaemonError::Internal( + "provider synchronization task ignored cancellation".to_string(), + )), + } + } + } + } + + async fn join(&mut self) -> Result<()> { + let Some(task) = self.task.as_mut() else { + return Ok(()); + }; + let result = task.await; + self.task.take(); + result.map_err(join_error) + } + + #[cfg(test)] + async fn wait_started(&self) { + self.started.notified().await; + } +} + +impl Drop for FlushLoop { + fn drop(&mut self) { + self.cancellation.cancel(); + if let Some(task) = self.task.as_ref() { + task.abort(); + } + } +} + +fn join_error(error: tokio::task::JoinError) -> BlazeDaemonError { + BlazeDaemonError::Internal(format!( + "provider synchronization task join failed: {error}" + )) +} + +/// Counters emitted for one synchronization sweep. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) struct FlushSummary { + /// Running records selected before any provider await. + pub(crate) selected: usize, + /// Provider calls that completed successfully. + pub(crate) flushed: usize, + /// Records that stopped being Running before their operation lock won. + pub(crate) skipped: usize, + /// Invalid owners and provider failures isolated from the rest of the sweep. + pub(crate) failed: usize, +} + +enum FlushAttempt { + Flushed, + Skipped, + Cancelled, +} + +impl SandboxManager { + /// Start a cancellable periodic storage synchronization worker. + /// + /// The first sweep starts after one complete interval. Missed ticks are + /// skipped instead of being queued behind a slow sweep. + pub(crate) fn start_flush_loop( + self: &Arc, + interval: Duration, + attempt_timeout: Duration, + ) -> FlushLoop { + let manager = self.clone(); + let cancellation = CancellationToken::new(); + let worker_cancellation = cancellation.clone(); + #[cfg(test)] + let started = Arc::new(Notify::new()); + #[cfg(test)] + let worker_started = started.clone(); + tracing::info!( + interval_secs = interval.as_secs_f64(), + attempt_timeout_secs = attempt_timeout.as_secs_f64(), + "starting provider synchronization task" + ); + let task = tokio::spawn(async move { + let first_tick = Instant::now() + interval; + let mut ticker = tokio::time::interval_at(first_tick, interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + #[cfg(test)] + worker_started.notify_one(); + loop { + tokio::select! { + biased; + _ = worker_cancellation.cancelled() => break, + _ = ticker.tick() => { + let summary = manager + .flush_all_until(&worker_cancellation, attempt_timeout) + .await; + let Some(summary) = summary else { + break; + }; + tracing::debug!( + selected = summary.selected, + flushed = summary.flushed, + skipped = summary.skipped, + failed = summary.failed, + "provider synchronization sweep completed" + ); + } + } + } + tracing::info!("provider synchronization task stopped"); + }); + FlushLoop { + cancellation, + task: Some(task), + #[cfg(test)] + started, + } + } + + #[cfg(test)] + async fn flush_all(&self, attempt_timeout: Duration) -> FlushSummary { + self.flush_all_until(&CancellationToken::new(), attempt_timeout) + .await + .expect("uncancelled sweep") + } + + async fn flush_all_until( + &self, + cancellation: &CancellationToken, + attempt_timeout: Duration, + ) -> Option { + let running_ids = match self.list() { + Ok(instances) => instances + .into_iter() + .filter_map(|instance| { + (instance.state == SandboxState::Running).then_some(instance.id) + }) + .collect::>(), + Err(error) => { + tracing::error!(%error, "cannot select provider synchronization candidates"); + return Some(FlushSummary { + failed: 1, + ..FlushSummary::default() + }); + } + }; + let mut summary = FlushSummary { + selected: running_ids.len(), + ..FlushSummary::default() + }; + for id in running_ids { + match self + .flush_if_running(id, cancellation, attempt_timeout) + .await + { + Ok(FlushAttempt::Flushed) => summary.flushed += 1, + Ok(FlushAttempt::Skipped) => summary.skipped += 1, + Ok(FlushAttempt::Cancelled) => return None, + Err(error) => { + summary.failed += 1; + tracing::warn!( + sandbox_id = %id, + %error, + "sandbox provider synchronization failed" + ); + } + } + } + Some(summary) + } + + async fn flush_if_running( + &self, + id: Uuid, + cancellation: &CancellationToken, + attempt_timeout: Duration, + ) -> Result { + let operation_lock = self.operation_lock(id); + let _operation = tokio::select! { + biased; + _ = cancellation.cancelled() => return Ok(FlushAttempt::Cancelled), + operation = operation_lock.lock() => operation, + }; + let instance = self.get(id)?; + if instance.state != SandboxState::Running { + return Ok(FlushAttempt::Skipped); + } + if let Some(operation) = instance.operation { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running with unfinished {} operation", + operation.kind + ))); + } + if instance.backend_ownership != BackendOwnership::Running { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running with {} backend ownership", + format!("{:?}", instance.backend_ownership).to_lowercase() + ))); + } + let backend = self.backend_owner(id).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} is Running without a backend owner" + )) + })?; + match backend.try_wait().await? { + None => {} + Some(status) => { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "instance {id} backend already exited: {status:?}" + ))); + } + } + let storage = self.reconstruct_storage(id).await.map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "instance {id} has no complete storage owner: {error}" + )) + })?; + tokio::select! { + biased; + _ = cancellation.cancelled() => Ok(FlushAttempt::Cancelled), + result = tokio::time::timeout(attempt_timeout, self.flush_storage(&storage)) => { + match result { + Ok(result) => { + result?; + Ok(FlushAttempt::Flushed) + } + Err(_) => Err(BlazeDaemonError::Internal(format!( + "provider synchronization for {id} timed out after {:.3} seconds", + attempt_timeout.as_secs_f64() + ))), + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::{HashMap, HashSet}; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + + use async_trait::async_trait; + use blaze_core::backend::{BackendKind, SpawnRequest}; + use blaze_core::error::{BlazeError, Result as CoreResult}; + use blaze_core::lifecycle::{ + BackendOwnership, OperationKind, SandboxInstance, SandboxState, StartPath, + }; + use blaze_core::policy::{BackendConfigs, WorkloadClass}; + use blaze_core::pool::PoolManager; + use blaze_core::storage::{ + AcquireOpts, PoolStatus, StorageAcquireError, StorageProvider, StorageSlot, + }; + use tokio::sync::Notify; + + use crate::file_provider::FileStorageProvider; + use crate::sandbox::manager::{SandboxManagerInit, SandboxManagerResources}; + use crate::spawner::{BackendSpawner, MockSpawner, SpawnerRegistry}; + + use super::*; + + struct RecordingStorage { + inner: FileStorageProvider, + instances: PathBuf, + calls: Mutex>, + call_recorded: Notify, + failures: Mutex>, + block_next: AtomicBool, + started: Notify, + } + + impl RecordingStorage { + fn new(images: PathBuf, instances: PathBuf) -> Self { + Self { + inner: FileStorageProvider::with_images(images, instances.clone()), + instances, + calls: Mutex::new(Vec::new()), + call_recorded: Notify::new(), + failures: Mutex::new(HashSet::new()), + block_next: AtomicBool::new(false), + started: Notify::new(), + } + } + + fn calls(&self) -> Vec { + self.calls.lock().expect("calls").clone() + } + + async fn wait_for_calls(&self, expected: usize) { + loop { + let call_recorded = self.call_recorded.notified(); + if self.calls.lock().expect("calls").len() >= expected { + return; + } + call_recorded.await; + } + } + + fn fail(&self, id: Uuid) { + self.failures + .lock() + .expect("failures") + .insert(id.to_string()); + } + + fn block_once(&self) { + self.block_next.store(true, Ordering::Release); + } + } + + #[async_trait] + impl StorageProvider for RecordingStorage { + async fn probe(&self) -> CoreResult { + self.inner.probe().await + } + + async fn acquire( + &self, + opts: &AcquireOpts, + ) -> std::result::Result { + self.inner.acquire(opts).await + } + + async fn release(&self, slot: StorageSlot) -> CoreResult<()> { + self.inner.release(slot).await + } + + async fn release_by_id(&self, instance_id: &str) -> CoreResult<()> { + self.inner.release_by_id(instance_id).await + } + + async fn reconstruct(&self, instance_id: &str) -> CoreResult { + self.inner.reconstruct(instance_id).await + } + + async fn flush_dirty(&self, slot: &StorageSlot) -> CoreResult<()> { + self.calls.lock().expect("calls").push(slot.id.clone()); + self.call_recorded.notify_waiters(); + if self.block_next.swap(false, Ordering::AcqRel) { + self.started.notify_one(); + std::future::pending::<()>().await; + } + if self.failures.lock().expect("failures").contains(&slot.id) { + return Err(BlazeError::StorageError { + msg: format!("injected provider synchronization failure for {}", slot.id), + }); + } + Ok(()) + } + + fn pool_status(&self) -> PoolStatus { + self.inner.pool_status() + } + + async fn drain_pool(&self) -> CoreResult { + self.inner.drain_pool().await + } + } + + fn manager( + temp: &Path, + storage: Arc, + ) -> (Arc, SandboxManagerResources) { + let state_dir = temp.join("state"); + let images = temp.join("images"); + let instances = temp.join("instances"); + for directory in [&state_dir, &images, &instances] { + std::fs::create_dir_all(directory).expect("test directory"); + } + let mut spawners = SpawnerRegistry::new(); + spawners.insert(BackendKind::Mock, Arc::new(MockSpawner)); + let (manager, resources) = SandboxManager::new(SandboxManagerInit { + instances: HashMap::new(), + pool: PoolManager::new(), + spawners, + active_backend: BackendKind::Mock, + storage, + state_dir, + rootfs_size: 64, + mem_size: 32, + }); + (Arc::new(manager), resources) + } + + async fn insert_running( + manager: &SandboxManager, + resources: &SandboxManagerResources, + storage: &RecordingStorage, + id: Uuid, + acquire_storage: bool, + insert_backend: bool, + active_operation: bool, + ) { + let slot = if acquire_storage { + Some( + storage + .acquire(&AcquireOpts { + instance_id: id.to_string(), + rootfs_size: 64, + mem_size: 32, + }) + .await + .expect("slot"), + ) + } else { + None + }; + let mut metadata = SandboxInstance::new( + BackendKind::Mock, + WorkloadClass::AgentTool, + "sha256:flush-test".into(), + StartPath::Cold, + "flush-test".into(), + ); + metadata.id = id; + metadata + .transition(SandboxState::Creating) + .expect("pending to creating"); + metadata + .transition(SandboxState::Running) + .expect("creating to running"); + metadata.backend_ownership = BackendOwnership::Running; + if active_operation { + metadata + .begin_operation(OperationKind::Create) + .expect("active operation"); + } + resources + .instances + .lock() + .expect("instances") + .insert(id, metadata); + if insert_backend { + let slot = slot.clone().unwrap_or_else(|| StorageSlot { + id: id.to_string(), + rootfs_path: PathBuf::new(), + mem_path: PathBuf::new(), + mem_diff_path: PathBuf::new(), + rootfs_diff_path: PathBuf::new(), + instance_dir: PathBuf::new(), + }); + let owner = MockSpawner + .spawn(SpawnRequest { + instance_id: id, + run_dir: storage.instances.join(id.to_string()).join("runtime"), + binary_path: PathBuf::new(), + storage: slot, + backend: BackendConfigs::default(), + vm: None, + }) + .await + .expect("mock owner"); + manager + .insert_backend_owner(id, owner) + .expect("register owner"); + } + } + + async fn settle() { + for _ in 0..8 { + tokio::task::yield_now().await; + } + } + + #[tokio::test] + async fn sweep_flushes_running_records_and_isolates_failures() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let failing = Uuid::new_v4(); + let succeeding = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, failing, true, true, false).await; + insert_running( + &manager, &resources, &storage, succeeding, true, true, false, + ) + .await; + storage.fail(failing); + + let summary = manager.flush_all(Duration::from_secs(1)).await; + + assert_eq!( + summary, + FlushSummary { + selected: 2, + flushed: 1, + skipped: 0, + failed: 1, + } + ); + assert_eq!( + storage.calls().into_iter().collect::>(), + HashSet::from([failing.to_string(), succeeding.to_string()]) + ); + } + + #[tokio::test] + async fn sweep_reports_incomplete_running_owners_without_flushing() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + true, + false, + false, + ) + .await; + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + false, + true, + false, + ) + .await; + insert_running( + &manager, + &resources, + &storage, + Uuid::new_v4(), + true, + true, + true, + ) + .await; + + assert_eq!( + manager.flush_all(Duration::from_secs(1)).await, + FlushSummary { + selected: 3, + flushed: 0, + skipped: 0, + failed: 3, + } + ); + assert!(storage.calls().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn timed_out_provider_call_keeps_slot_retryable() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + storage.block_once(); + + let first = { + let manager = manager.clone(); + tokio::spawn(async move { manager.flush_all(Duration::from_secs(5)).await }) + }; + storage.started.notified().await; + tokio::time::advance(Duration::from_secs(5)).await; + assert_eq!(first.await.expect("sweep").failed, 1); + + let second = manager.flush_all(Duration::from_secs(5)).await; + assert_eq!(second.flushed, 1); + assert_eq!(storage.calls(), vec![id.to_string(), id.to_string()]); + } + + #[tokio::test] + async fn sweep_waits_for_operation_lock_and_rechecks_state() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + let lock = manager.operation_lock(id); + let guard = lock.lock().await; + let sweep = { + let manager = manager.clone(); + tokio::spawn(async move { manager.flush_all(Duration::from_secs(1)).await }) + }; + settle().await; + resources + .instances + .lock() + .expect("instances") + .get_mut(&id) + .expect("metadata") + .transition(SandboxState::RecoveryRequired) + .expect("running to recovery-required"); + drop(guard); + + assert_eq!(sweep.await.expect("sweep").skipped, 1); + assert!(storage.calls().is_empty()); + } + + #[tokio::test(start_paused = true)] + async fn periodic_worker_delays_first_tick_and_stops_before_next_sweep() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + + let mut worker = manager.start_flush_loop(Duration::from_secs(10), Duration::from_secs(5)); + worker.wait_started().await; + settle().await; + assert!(storage.calls().is_empty()); + tokio::time::advance(Duration::from_secs(10)).await; + storage.wait_for_calls(1).await; + assert_eq!(storage.calls(), vec![id.to_string()]); + + worker.shutdown().await.expect("worker shutdown"); + tokio::time::advance(Duration::from_secs(30)).await; + settle().await; + assert_eq!(storage.calls(), vec![id.to_string()]); + } + + #[tokio::test(start_paused = true)] + async fn worker_shutdown_cancels_a_sweep_waiting_for_operation_lock() { + let temp = tempfile::tempdir().expect("temp"); + let storage = Arc::new(RecordingStorage::new( + temp.path().join("images"), + temp.path().join("instances"), + )); + let (manager, resources) = manager(temp.path(), storage.clone()); + let id = Uuid::new_v4(); + insert_running(&manager, &resources, &storage, id, true, true, false).await; + let lock = manager.operation_lock(id); + let guard = lock.lock().await; + let mut worker = manager.start_flush_loop(Duration::from_secs(10), Duration::from_secs(5)); + worker.wait_started().await; + tokio::time::advance(Duration::from_secs(10)).await; + settle().await; + + worker.shutdown().await.expect("worker shutdown"); + drop(guard); + assert!(storage.calls().is_empty()); + } + + #[tokio::test] + async fn supervisor_reports_an_unexpected_worker_exit() { + let cancellation = CancellationToken::new(); + let mut worker = FlushLoop { + cancellation, + task: Some(tokio::spawn(async {})), + started: Arc::new(Notify::new()), + }; + + let error = worker + .observe_exit() + .await + .expect_err("early worker exit must stop the daemon"); + + assert!(error.to_string().contains("exited unexpectedly")); + worker + .shutdown() + .await + .expect("finished worker is already joined"); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 362daedd27..983fcb3ad0 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -180,14 +180,24 @@ impl SandboxManager { Ok(operation) } - #[cfg(test)] - pub fn backend_owner(&self, id: Uuid) -> Option { + pub(crate) fn backend_owner(&self, id: Uuid) -> Option { match self.backend_instances.lock() { Ok(instances) => instances.get(&id).cloned(), Err(poisoned) => poisoned.into_inner().get(&id).cloned(), } } + pub(super) async fn reconstruct_storage(&self, id: Uuid) -> Result { + self.storage + .reconstruct(&id.to_string()) + .await + .map_err(Into::into) + } + + pub(super) async fn flush_storage(&self, slot: &StorageSlot) -> Result<()> { + self.storage.flush_dirty(slot).await.map_err(Into::into) + } + #[cfg(test)] pub fn insert_backend_owner(&self, id: Uuid, backend: DynBackendInstance) -> Result<()> { self.backend_instances diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index be8b5cf5b1..670dc55bc5 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -36,7 +36,7 @@ pub struct ServerState { pub hook: Mutex, #[cfg(test)] pub instances: Arc>>, - pub manager: SandboxManager, + pub manager: Arc, /// The backend kind that `build_spawner` actually probed and selected. /// API handlers use this to constrain availability to the single active /// backend rather than reporting all configured binaries. @@ -82,7 +82,7 @@ impl ServerState { hook: Mutex::new(hook), #[cfg(test)] instances: resources.instances, - manager, + manager: Arc::new(manager), active_backend, storage, #[cfg(test)] From 515b5e2933dc248068fd05c211b8b62988a95c7e Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 15:20:47 +0800 Subject: [PATCH 36/40] docs(blaze): explain storage synchronization This explains how operators opt into periodic provider synchronization, which running sandboxes are eligible, and how one failed or timed-out slot remains owned and retryable without blocking the rest of a sweep. The English and Chinese guides also document missed-tick behavior, the provider cancellation contract, and the worker, connection, and runtime shutdown order. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 10 ++- src/blaze/README_zh.md | 9 ++- .../docs/design/storage-synchronization.md | 65 +++++++++++++++++++ .../docs/design/storage-synchronization_zh.md | 58 +++++++++++++++++ src/blaze/examples/config.toml | 8 ++- 5 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 src/blaze/docs/design/storage-synchronization.md create mode 100644 src/blaze/docs/design/storage-synchronization_zh.md diff --git a/src/blaze/README.md b/src/blaze/README.md index e8988d66f5..580b9ab46e 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -103,10 +103,18 @@ provider = "file" # Storage provider selection. Currently supported: "file images_dir = "/var/lib/blaze/images" # pool_size = 0 # [Reserved] Warm pool slots (not yet active) # prefork = false # [Reserved] Pre-start VMs in pool (not yet active) -# flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) +flush_interval = "disabled" # Set a positive duration to synchronize running slots. +flush_timeout = "30s" # Maximum duration of one provider synchronization attempt. ``` The `file` provider uses standard filesystem operations for sandbox storage. The `auto` provider probes available backends in priority order (currently equivalent to `file`). Unrecognized values will log a warning and fall back to `file`. +When periodic synchronization is enabled, one provider failure or timeout does +not block later sandboxes. The slot remains owned so a later sweep or destroy +can retry it. The daemon stops and joins the synchronization worker before +draining connections and releasing runtime resources. + +See [Storage Synchronization](docs/design/storage-synchronization.md) for +selection, retry, and shutdown behavior. ## API Endpoints diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index ff96fa68e7..1af436a549 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -101,10 +101,17 @@ provider = "file" # 存储 provider 选择。当前支持："file"、"auto images_dir = "/var/lib/blaze/images" # pool_size = 0 # [Reserved] 预热存储槽位数（尚未启用） # prefork = false # [Reserved] 是否在槽位中预启动 VM（尚未启用） -# flush_interval = "30s" # [Reserved] 脏数据刷盘周期（尚未启用） +flush_interval = "disabled" # 设置正数 duration 后同步 running slot。 +flush_timeout = "30s" # 单次 provider 同步的最长时间。 ``` `file` provider 使用标准文件系统操作管理 sandbox 存储。`auto` 按优先级探测可用 provider（当前等同于 `file`）。无法识别的值将记录告警并回退到 `file`。 +启用周期同步后，单个 provider 失败或超时不会阻塞后续 sandbox。slot 会继续 +由 daemon 持有，后续 sweep 或 destroy 可以重试。daemon 会先停止并等待同步 +任务退出，再排空连接和释放 runtime 资源。 + +[存储同步](docs/design/storage-synchronization_zh.md)进一步说明选择、重试和 +关闭行为。 ## API 端点 diff --git a/src/blaze/docs/design/storage-synchronization.md b/src/blaze/docs/design/storage-synchronization.md new file mode 100644 index 0000000000..a3fd753b5a --- /dev/null +++ b/src/blaze/docs/design/storage-synchronization.md @@ -0,0 +1,65 @@ +# Storage Synchronization + +[中文版](storage-synchronization_zh.md) + +Blaze can periodically ask the configured `StorageProvider` to synchronize +data owned by running sandboxes. This closes the gap between a provider that +can synchronize one slot and a daemon that schedules the operation safely +across all eligible sandboxes. + +Periodic synchronization is disabled by default. Set +`storage.flush_interval` to a positive duration to enable it. +`storage.flush_timeout` is a positive duration that bounds each provider call +and defaults to 30 seconds. + +## Which sandboxes are synchronized + +At the beginning of a sweep, the manager selects records whose lifecycle state +is `Running`. Before it calls the provider for one sandbox, it enters the same +operation lock used by create and destroy and checks the record again. + +The provider call runs only when all of these conditions still hold: + +- the lifecycle state is `Running`; +- there is no unfinished lifecycle operation; +- metadata says the backend is running and the daemon still owns that backend; +- the provider can reconstruct a complete slot from the sandbox ID. + +A sandbox that changed state while waiting for the operation lock is skipped. +An inconsistent Running record is reported as a failed item instead of being +silently omitted. The remaining sandboxes in the sweep still run. + +The first sweep starts after one complete interval. Missed ticks are skipped +instead of queued, so a slow sweep cannot create an unbounded backlog. + +## Failure and retry behavior + +Each provider call has its own deadline. A failure or timeout leaves the slot +owned by the sandbox and does not change lifecycle state. A later sweep or +destroy can therefore retry the provider operation. + +`StorageProvider::flush_dirty` is the provider-specific persistence boundary. +Implementations must leave a cancelled call safe to retry or release. The file +provider synchronizes the canonical files in its independent sandbox slot. +Other providers can use a different mechanism while preserving the same +ownership and cancellation contract. + +Storage synchronization does not save VM memory or device state. It is not a +substitute for saving and restoring a complete runtime. + +## Daemon shutdown + +The daemon supervises the periodic worker while serving requests. If the +worker exits unexpectedly, the daemon stops accepting work and follows the +normal coordinated shutdown path. + +During normal shutdown, the daemon performs these steps in order: + +1. stop accepting new connections; +2. cancel and join the synchronization worker; +3. drain accepted connections; +4. release owned runtime and storage resources. + +The worker is gone before destroy starts, so a periodic provider call cannot +race with teardown of the same sandbox. If both the worker and a later +shutdown stage fail, the daemon reports both failures. diff --git a/src/blaze/docs/design/storage-synchronization_zh.md b/src/blaze/docs/design/storage-synchronization_zh.md new file mode 100644 index 0000000000..366de3e221 --- /dev/null +++ b/src/blaze/docs/design/storage-synchronization_zh.md @@ -0,0 +1,58 @@ +# 存储同步 + +[English](storage-synchronization.md) + +Blaze 可以定期要求已配置的 `StorageProvider` 同步 running sandbox 持有的 +数据。这样，provider 不仅具备同步单个 slot 的能力，daemon 也能安全调度 +所有符合条件的 sandbox。 + +周期同步默认关闭。将 `storage.flush_interval` 设置为正数 duration 后启用。 +`storage.flush_timeout` 是单次 provider 调用的最长时间，默认值为 30 秒。 + +## 哪些 sandbox 会被同步 + +每轮开始时，manager 先选择 lifecycle 状态为 `Running` 的记录。调用某个 +sandbox 的 provider 前，会取得 create 和 destroy 共用的 operation lock， +然后重新检查记录。 + +只有同时满足以下条件时才会调用 provider： + +- lifecycle 状态仍为 `Running`； +- 没有未结束的 lifecycle operation； +- metadata 记录 backend 正在运行，而且 daemon 仍持有该 backend； +- provider 可以根据 sandbox ID 重建完整 slot。 + +等待 operation lock 期间改变状态的 sandbox 会被跳过。状态为 Running 但 +ownership 不完整的记录会计为失败，而不是被静默遗漏；本轮的其他 sandbox +仍会继续处理。 + +第一次 sweep 在一个完整 interval 后开始。错过的 tick 会被跳过，不会排队， +因此耗时较长的一轮不会形成无界积压。 + +## 失败与重试 + +每次 provider 调用都有独立 deadline。失败或超时后，slot 仍归 sandbox +持有，lifecycle 状态也不会改变；后续 sweep 或 destroy 可以再次尝试。 + +`StorageProvider::flush_dirty` 是 provider 特定的持久化边界。实现必须保证 +调用被取消后仍可安全重试或释放。file provider 会同步独立 sandbox slot +中的规范文件；其他 provider 可以采用不同机制，但必须保持相同的 ownership +和取消合同。 + +存储同步不会保存 VM 内存或设备状态，不能代替完整 runtime 的保存和恢复。 + +## Daemon 关闭 + +daemon 提供请求服务时会同时监控周期 worker。如果 worker 意外退出，daemon +会停止接收工作，并进入正常的协调关闭流程。 + +正常关闭按以下顺序进行： + +1. 停止接收新连接； +2. 取消并等待同步 worker 退出； +3. 排空已经接收的连接； +4. 释放仍持有的 runtime 和 storage 资源。 + +destroy 开始前 worker 已经退出，因此周期 provider 调用不会和同一 +sandbox 的清理并发。如果 worker 与后续关闭阶段都失败，daemon 会同时报告 +两项失败。 diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 1b1bc127d7..ead1ab86cc 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -102,8 +102,12 @@ mem_size = 4294967296 # Whether to prefork (pre-allocate) storage slots at daemon startup. # prefork = false # [Reserved] Pre-start VMs in pool (not yet active) -# Interval between flush-dirty sweeps. -# flush_interval = "30s" # [Reserved] Dirty data flush period (not yet active) +# Periodic provider synchronization is opt-in. Set a positive duration to +# synchronize running sandbox slots after each interval. +flush_interval = "disabled" + +# Maximum duration of one provider synchronization attempt. +flush_timeout = "30s" # --------------------------------------------------------------------------- # Instance pool settings (global defaults for warm-pool recycling). From 625709b2e163d46d0f280fc01223fa3b16b0f8da Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 17:19:41 +0800 Subject: [PATCH 37/40] feat(blaze): publish runtime artifact catalog This enables operators to import, list, and inspect durable runtime artifact sets through /v1/runtime-templates. Imports are constrained to a configured source root, copied into private staging directories, and published atomically. Capacity reservations, source revalidation, and shutdown coordination prevent incomplete data from becoming visible. If durability or cleanup becomes uncertain, later imports stop until the catalog is repaired and the daemon restarts. Sandbox creation does not consume catalog entries in this commit. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/config.rs | 213 ++- src/blaze/crates/blazed/src/api.rs | 173 ++ src/blaze/crates/blazed/src/daemon.rs | 70 +- src/blaze/crates/blazed/src/error.rs | 4 + src/blaze/crates/blazed/src/sandbox.rs | 1 + .../crates/blazed/src/sandbox/manager.rs | 5 + .../crates/blazed/src/sandbox/template.rs | 1565 +++++++++++++++++ src/blaze/crates/blazed/src/state.rs | 3 + src/blaze/examples/config.toml | 17 + 9 files changed, 2039 insertions(+), 12 deletions(-) create mode 100644 src/blaze/crates/blazed/src/sandbox/template.rs diff --git a/src/blaze/crates/blaze-core/src/config.rs b/src/blaze/crates/blaze-core/src/config.rs index 2c490167a2..c406b7f320 100644 --- a/src/blaze/crates/blaze-core/src/config.rs +++ b/src/blaze/crates/blaze-core/src/config.rs @@ -30,6 +30,8 @@ pub struct DaemonConfig { #[serde(default)] pub template: TemplateSection, #[serde(default)] + pub runtime_templates: RuntimeTemplateSection, + #[serde(default)] pub metrics: MetricsSection, } @@ -139,6 +141,45 @@ impl Default for TemplateSection { } } +/// Published runtime artifact catalog and its local import boundary. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RuntimeTemplateSection { + /// Directory containing atomically published runtime artifact sets. + #[serde(default = "default_runtime_template_dir")] + pub dir: PathBuf, + /// Optional root containing operator-prepared import sources. + /// + /// Imports are disabled when this value is absent. API callers provide a + /// relative path below this root rather than an arbitrary daemon path. + #[serde(default)] + pub import_root: Option, + /// Maximum number of regular files accepted from one source directory. + #[serde(default = "default_runtime_template_max_files")] + pub max_files: usize, + /// Maximum final artifact and generated metadata bytes for one import. + #[serde(default = "default_runtime_template_max_bytes")] + pub max_bytes: u64, + /// Maximum serialized size of one published `template.json`. + #[serde(default = "default_runtime_template_max_metadata_bytes")] + pub max_metadata_bytes: u64, + /// Maximum aggregate bytes retained by the published catalog. + #[serde(default = "default_runtime_template_max_total_bytes")] + pub max_total_bytes: u64, +} + +impl Default for RuntimeTemplateSection { + fn default() -> Self { + Self { + dir: default_runtime_template_dir(), + import_root: None, + max_files: default_runtime_template_max_files(), + max_bytes: default_runtime_template_max_bytes(), + max_metadata_bytes: default_runtime_template_max_metadata_bytes(), + max_total_bytes: default_runtime_template_max_total_bytes(), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MetricsSection { #[serde(default = "default_prometheus_socket")] @@ -226,7 +267,47 @@ impl DaemonConfig { ), }); } - validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir) + validate_storage_paths(&self.storage.images_dir, &self.storage.instances_dir)?; + validate_runtime_template_paths( + &self.runtime_templates.dir, + self.runtime_templates.import_root.as_deref(), + &self.storage.images_dir, + &self.storage.instances_dir, + &self.template.dir, + )?; + if self.runtime_templates.max_files == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_files must be greater than zero".to_string(), + ), + }); + } + if self.runtime_templates.max_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_bytes must be greater than zero".to_string(), + ), + }); + } + if self.runtime_templates.max_metadata_bytes == 0 + || self.runtime_templates.max_metadata_bytes > self.runtime_templates.max_bytes + { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_metadata_bytes must be greater than zero and no \ + larger than max_bytes" + .to_string(), + ), + }); + } + if self.runtime_templates.max_total_bytes == 0 { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue( + "runtime_templates.max_total_bytes must be greater than zero".to_string(), + ), + }); + } + Ok(()) } } @@ -247,6 +328,78 @@ pub fn validate_storage_paths(images_dir: &Path, instances_dir: &Path) -> Result Ok(()) } +fn validate_runtime_template_paths( + dir: &Path, + import_root: Option<&Path>, + images_dir: &Path, + instances_dir: &Path, + template_dir: &Path, +) -> Result<()> { + validate_absolute_root(dir, "runtime_templates.dir")?; + if let Some(import_root) = import_root { + validate_absolute_root(import_root, "runtime_templates.import_root")?; + } + + let mut roots = vec![ + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ("template.dir", template_dir), + ]; + if let Some(import_root) = import_root { + roots.push(("runtime_templates.import_root", import_root)); + } + for (label, root) in roots { + if paths_overlap(dir, root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime_templates.dir ({}) and {label} ({}) must be disjoint", + dir.display(), + root.display() + )), + }); + } + } + + if let Some(import_root) = import_root { + for (label, root) in [ + ("storage.images_dir", images_dir), + ("storage.instances_dir", instances_dir), + ("template.dir", template_dir), + ] { + if paths_overlap(import_root, root) { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "runtime_templates.import_root ({}) and {label} ({}) must be disjoint", + import_root.display(), + root.display() + )), + }); + } + } + } + Ok(()) +} + +fn validate_absolute_root(path: &Path, label: &str) -> Result<()> { + if !path.is_absolute() + || path + .components() + .any(|component| matches!(component, std::path::Component::ParentDir)) + { + return Err(BlazeError::ConfigError { + source: ConfigErrorSource::InvalidValue(format!( + "{label} ({}) must be an absolute path without parent components", + path.display() + )), + }); + } + Ok(()) +} + +fn paths_overlap(left: &Path, right: &Path) -> bool { + left == right || left.starts_with(right) || right.starts_with(left) +} + // ----- defaults ----- fn default_log_level() -> String { @@ -282,6 +435,21 @@ fn default_template_gc_interval() -> String { fn default_template_idle_ttl() -> String { "1h".to_string() } +fn default_runtime_template_dir() -> PathBuf { + PathBuf::from("/var/lib/blaze/runtime-templates") +} +fn default_runtime_template_max_files() -> usize { + 32 +} +fn default_runtime_template_max_bytes() -> u64 { + 256 * 1024 * 1024 * 1024 +} +fn default_runtime_template_max_metadata_bytes() -> u64 { + 1024 * 1024 +} +fn default_runtime_template_max_total_bytes() -> u64 { + 1024 * 1024 * 1024 * 1024 +} fn default_prometheus_socket() -> PathBuf { PathBuf::from("/run/blaze/metrics.sock") } @@ -316,6 +484,7 @@ mod tests { assert_eq!(cfg.api.max_body_bytes, 1024 * 1024); assert!(cfg.backends.is_empty()); assert_ne!(cfg.storage.images_dir, cfg.storage.instances_dir); + assert!(cfg.runtime_templates.import_root.is_none()); } #[test] @@ -384,4 +553,46 @@ mod tests { assert!(error.to_string().contains("must be disjoint")); } } + #[test] + fn rejects_unsafe_runtime_template_boundaries() { + let mut relative = DaemonConfig::default(); + relative.runtime_templates.dir = PathBuf::from("runtime-templates"); + assert!(relative.validate().is_err()); + + let mut parent = DaemonConfig::default(); + parent.runtime_templates.dir = PathBuf::from("/var/lib/blaze/../runtime-templates"); + assert!(parent.validate().is_err()); + + let mut overlapping = DaemonConfig::default(); + overlapping.runtime_templates.import_root = + Some(PathBuf::from("/var/lib/blaze/runtime-templates/imports")); + assert!(overlapping.validate().is_err()); + + for owned_root in [ + "/var/lib/blaze/images/catalog", + "/var/lib/blaze/instances/catalog", + "/var/lib/blaze/templates/catalog", + ] { + let mut config = DaemonConfig::default(); + config.runtime_templates.dir = PathBuf::from(owned_root); + assert!(config.validate().is_err()); + } + + let mut source_overlap = DaemonConfig::default(); + source_overlap.runtime_templates.import_root = + Some(PathBuf::from("/var/lib/blaze/images/imports")); + assert!(source_overlap.validate().is_err()); + + let mut unbounded = DaemonConfig::default(); + unbounded.runtime_templates.max_files = 0; + assert!(unbounded.validate().is_err()); + + let mut metadata = DaemonConfig::default(); + metadata.runtime_templates.max_metadata_bytes = metadata.runtime_templates.max_bytes + 1; + assert!(metadata.validate().is_err()); + + let mut total = DaemonConfig::default(); + total.runtime_templates.max_total_bytes = 0; + assert!(total.validate().is_err()); + } } diff --git a/src/blaze/crates/blazed/src/api.rs b/src/blaze/crates/blazed/src/api.rs index 54753d1dfd..ded6ba99e1 100644 --- a/src/blaze/crates/blazed/src/api.rs +++ b/src/blaze/crates/blazed/src/api.rs @@ -7,6 +7,7 @@ use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::convert::Infallible; +use std::path::PathBuf; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -100,6 +101,11 @@ async fn dispatch( ("POST", ["v1", "templates", "gc"]) => gc_templates(state), ("GET", ["v1", "templates"]) => list_templates(state), ("GET", ["v1", "templates", id]) => inspect_template(state, id), + ("GET", ["v1", "runtime-templates"]) => list_runtime_templates(state).await, + ("GET", ["v1", "runtime-templates", name]) => get_runtime_template(state, name).await, + ("POST", ["v1", "runtime-templates", "import"]) => { + import_runtime_template(state, &body).await + } ("GET", ["v1", "policies"]) => list_policies(state), ("GET", ["v1", "hooks"]) => list_hooks(state), ("GET", ["v1", "metrics"]) => metrics(state), @@ -591,6 +597,39 @@ fn inspect_template(state: &Arc, id: &str) -> Result) -> Result>> { + json_ok(&state.manager.list_runtime_templates().await?) +} + +async fn get_runtime_template( + state: &Arc, + name: &str, +) -> Result>> { + json_ok(&state.manager.get_runtime_template(name.to_string()).await?) +} + +#[derive(Debug, Deserialize)] +struct ImportRuntimeTemplateRequest { + name: String, + source: PathBuf, + #[serde(default)] + description: String, +} + +async fn import_runtime_template( + state: &Arc, + body: &[u8], +) -> Result>> { + let request: ImportRuntimeTemplateRequest = serde_json::from_slice(body).map_err(|error| { + BlazeDaemonError::BadRequest(format!("invalid runtime template import body: {error}")) + })?; + let imported = state + .manager + .import_runtime_template(request.name, request.source, request.description) + .await?; + json_response(StatusCode::CREATED, &imported) +} + fn gc_templates(state: &Arc) -> Result>> { let idle_ttl = { let cfg = state @@ -738,6 +777,7 @@ mod tests { config.daemon.state_dir = temp.path().join("state"); config.storage.images_dir = temp.path().join("images"); config.storage.instances_dir = temp.path().join("instances"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); std::fs::create_dir_all(&config.storage.images_dir).expect("images"); std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); @@ -1482,6 +1522,7 @@ mod tests { // Minimal config with both backends present. let mut config = DaemonConfig::default(); config.daemon.state_dir = tmp.join("state"); + config.runtime_templates.dir = tmp.join("runtime-templates"); let _ = std::fs::create_dir_all(&config.daemon.state_dir); config.backends.insert("firecracker".into(), fc_bin.clone()); config @@ -1800,6 +1841,7 @@ mod tests { config.daemon.state_dir = temp.path().join("state"); config.storage.images_dir = temp.path().join("images"); config.storage.instances_dir = temp.path().join("instances"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); std::fs::create_dir_all(&config.daemon.state_dir).expect("state"); std::fs::create_dir_all(&config.storage.images_dir).expect("images"); std::fs::create_dir_all(&config.storage.instances_dir).expect("instances"); @@ -3340,4 +3382,135 @@ mod tests { SandboxState::Destroyed ); } + + #[tokio::test] + async fn runtime_template_routes_import_list_and_get_published_artifacts() { + let temp = tempfile::tempdir().expect("temp"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + std::fs::create_dir(&import_root).expect("import root"); + std::fs::create_dir(&source).expect("source"); + std::fs::write(source.join("vmstate.snap"), b"snapshot").expect("snapshot"); + std::fs::write(source.join("mem.bin"), b"memory").expect("memory"); + std::fs::write(source.join("rootfs.ext4"), b"rootfs").expect("rootfs"); + + let mut config = DaemonConfig::default(); + config.daemon.state_dir = temp.path().join("state"); + config.storage.images_dir = temp.path().join("images"); + config.storage.instances_dir = temp.path().join("instances"); + config.template.dir = temp.path().join("templates"); + config.runtime_templates.dir = temp.path().join("runtime-templates"); + config.runtime_templates.import_root = Some(import_root); + for directory in [ + &config.daemon.state_dir, + &config.storage.images_dir, + &config.storage.instances_dir, + &config.template.dir, + &config.runtime_templates.dir, + ] { + std::fs::create_dir_all(directory).expect("directory"); + } + let storage: Arc = + Arc::new(FileStorageProvider::with_images( + config.storage.images_dir.clone(), + config.storage.instances_dir.clone(), + )); + let state = Arc::new( + ServerState::build( + config, + PolicyEngine::with_policies(Vec::new()), + PoolManager::new(), + TemplateRegistry::new(), + HookRegistry::new(), + spawners(BackendKind::Mock, Arc::new(MockSpawner)), + BackendKind::Mock, + storage, + ) + .expect("state"), + ); + + let request = serde_json::to_vec(&json!({ + "name": "runtime-base", + "source": "source", + "description": "reusable runtime", + })) + .expect("request"); + let imported = dispatch( + &Method::POST, + "/v1/runtime-templates/import", + "", + request.clone(), + &state, + ) + .await + .expect("import"); + assert_eq!(imported.status(), StatusCode::CREATED); + let imported = serde_json::from_slice::( + &imported + .into_body() + .collect() + .await + .expect("body") + .to_bytes(), + ) + .expect("json"); + assert_eq!(imported["name"], "runtime-base"); + assert_eq!(imported["description"], "reusable runtime"); + + let listed = dispatch( + &Method::GET, + "/v1/runtime-templates", + "", + Vec::new(), + &state, + ) + .await + .expect("list"); + let listed = serde_json::from_slice::( + &listed.into_body().collect().await.expect("body").to_bytes(), + ) + .expect("json"); + assert_eq!(listed.as_array().expect("templates").len(), 1); + assert_eq!(listed[0]["name"], "runtime-base"); + + let fetched = dispatch( + &Method::GET, + "/v1/runtime-templates/runtime-base", + "", + Vec::new(), + &state, + ) + .await + .expect("get"); + let fetched = serde_json::from_slice::( + &fetched + .into_body() + .collect() + .await + .expect("body") + .to_bytes(), + ) + .expect("json"); + assert_eq!(fetched, imported); + + let duplicate = dispatch( + &Method::POST, + "/v1/runtime-templates/import", + "", + request, + &state, + ) + .await + .expect_err("duplicate"); + assert!(matches!(duplicate, BlazeDaemonError::Conflict(_))); + + let legacy = dispatch(&Method::GET, "/v1/templates", "", Vec::new(), &state) + .await + .expect("legacy template registry"); + let legacy = serde_json::from_slice::( + &legacy.into_body().collect().await.expect("body").to_bytes(), + ) + .expect("json"); + assert_eq!(legacy, json!([])); + } } diff --git a/src/blaze/crates/blazed/src/daemon.rs b/src/blaze/crates/blazed/src/daemon.rs index 2a30ddf85f..bc1da80bd4 100644 --- a/src/blaze/crates/blazed/src/daemon.rs +++ b/src/blaze/crates/blazed/src/daemon.rs @@ -397,27 +397,51 @@ async fn serve( uds.stop_accepting(); drop(tcp); + state.manager.cancel_runtime_template_imports(); let drain = connections.shutdown(SHUTDOWN_BUDGET.connection_drain); + let imports = state.manager.wait_for_runtime_template_imports(); let cleanup = api::shutdown_instances(&state, SHUTDOWN_BUDGET.runtime_cleanup); - let shutdown_result = finish_shutdown(drain, cleanup).await; + let shutdown_result = finish_shutdown(drain, imports, cleanup).await; tracing::info!("blaze daemon stopped"); shutdown_result } -async fn finish_shutdown(drain: Drain, cleanup: Cleanup) -> Result<()> +async fn finish_shutdown( + drain: Drain, + imports: Imports, + cleanup: Cleanup, +) -> Result<()> where Drain: Future>, + Imports: Future>, Cleanup: Future>, { let drain_result = drain.await; + let import_result = imports.await; let cleanup_result = cleanup.await; - match (drain_result, cleanup_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), - (Err(drain), Err(cleanup)) => Err(BlazeDaemonError::RecoveryRequired(format!( - "connection drain failed: {drain}; runtime cleanup failed: {cleanup}" - ))), + let mut failures = Vec::new(); + if let Err(error) = drain_result { + failures.push(("connection drain", error)); } + if let Err(error) = import_result { + failures.push(("runtime template import shutdown", error)); + } + if let Err(error) = cleanup_result { + failures.push(("runtime cleanup", error)); + } + if failures.is_empty() { + return Ok(()); + } + if failures.len() == 1 { + return Err(failures.remove(0).1); + } + Err(BlazeDaemonError::RecoveryRequired( + failures + .into_iter() + .map(|(stage, error)| format!("{stage} failed: {error}")) + .collect::>() + .join("; "), + )) } async fn serve_connection( @@ -589,14 +613,38 @@ mod tests { cleanup_observed.store(true, Ordering::Release); Ok(()) }; - let error = finish_shutdown(connections.shutdown(Duration::from_millis(10)), cleanup) - .await - .expect_err("connection timeout must be reported"); + let error = finish_shutdown( + connections.shutdown(Duration::from_millis(10)), + future::ready(Ok(())), + cleanup, + ) + .await + .expect_err("connection timeout must be reported"); assert!(error.to_string().contains("connection drain timed out")); assert!(cleanup_started.load(Ordering::Acquire)); } + #[tokio::test] + async fn shutdown_preserves_import_and_runtime_failures() { + let error = finish_shutdown( + future::ready(Ok(())), + future::ready(Err(BlazeDaemonError::Internal( + "import did not stop".to_string(), + ))), + future::ready(Err(BlazeDaemonError::Internal( + "runtime cleanup failed".to_string(), + ))), + ) + .await + .expect_err("both failures must be reported"); + let message = error.to_string(); + + assert!(message.contains("runtime template import shutdown failed")); + assert!(message.contains("import did not stop")); + assert!(message.contains("runtime cleanup failed")); + } + #[test] fn service_stop_timeout_covers_shutdown_stages() { let unit = include_str!("../../../dist/blazed.service"); diff --git a/src/blaze/crates/blazed/src/error.rs b/src/blaze/crates/blazed/src/error.rs index 560783115c..e9f33e4a71 100644 --- a/src/blaze/crates/blazed/src/error.rs +++ b/src/blaze/crates/blazed/src/error.rs @@ -54,6 +54,9 @@ pub enum BlazeDaemonError { #[error("conflict: {0}")] Conflict(String), + #[error("service unavailable: {0}")] + ServiceUnavailable(String), + #[error("unsupported operation: {0}")] UnsupportedOperation(String), @@ -100,6 +103,7 @@ impl BlazeDaemonError { BlazeDaemonError::BadRequest(_) | BlazeDaemonError::RequestBody(_) => 400, BlazeDaemonError::NotFound(_) => 404, BlazeDaemonError::Conflict(_) => 409, + BlazeDaemonError::ServiceUnavailable(_) => 503, BlazeDaemonError::PayloadTooLarge { .. } => 413, BlazeDaemonError::UnsupportedOperation(_) => 501, BlazeDaemonError::RecoveryRequired(_) => 500, diff --git a/src/blaze/crates/blazed/src/sandbox.rs b/src/blaze/crates/blazed/src/sandbox.rs index 2952f6f3ab..33d397b725 100644 --- a/src/blaze/crates/blazed/src/sandbox.rs +++ b/src/blaze/crates/blazed/src/sandbox.rs @@ -2,5 +2,6 @@ //! Managed sandbox lifecycle and runtime ownership. mod manager; +pub(crate) mod template; pub use manager::{CreateSandbox, SandboxManager, SandboxManagerInit}; diff --git a/src/blaze/crates/blazed/src/sandbox/manager.rs b/src/blaze/crates/blazed/src/sandbox/manager.rs index 362daedd27..82f7413d68 100644 --- a/src/blaze/crates/blazed/src/sandbox/manager.rs +++ b/src/blaze/crates/blazed/src/sandbox/manager.rs @@ -19,6 +19,7 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::metrics::Metrics; +use crate::sandbox::template::RuntimeTemplateCatalog; use crate::spawner::{DynBackendInstance, SpawnerRegistry}; const INSTANCE_CLEANUP_TIMEOUT: Duration = Duration::from_secs(30); @@ -82,6 +83,7 @@ pub struct SandboxManager { rootfs_size: u64, mem_size: u64, metrics: Arc, + pub(super) runtime_templates: RuntimeTemplateCatalog, } /// Construction inputs grouped to keep daemon wiring explicit. @@ -94,6 +96,7 @@ pub struct SandboxManagerInit { pub state_dir: PathBuf, pub rootfs_size: u64, pub mem_size: u64, + pub runtime_templates: RuntimeTemplateCatalog, } /// Shared read-only resources retained by [`crate::state::ServerState`]. @@ -116,6 +119,7 @@ impl SandboxManager { state_dir, rootfs_size, mem_size, + runtime_templates, } = init; let instances = Arc::new(Mutex::new(instances)); let backend_instances = Arc::new(Mutex::new(HashMap::new())); @@ -140,6 +144,7 @@ impl SandboxManager { rootfs_size, mem_size, metrics, + runtime_templates, }, resources, ) diff --git a/src/blaze/crates/blazed/src/sandbox/template.rs b/src/blaze/crates/blazed/src/sandbox/template.rs new file mode 100644 index 0000000000..7a30b55a29 --- /dev/null +++ b/src/blaze/crates/blazed/src/sandbox/template.rs @@ -0,0 +1,1565 @@ +// SPDX-License-Identifier: Apache-2.0 +//! Durable runtime artifact publication and lookup. + +use std::collections::HashSet; +use std::ffi::{CStr, CString, OsStr, OsString}; +use std::fs::{DirBuilder, File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::{AsRawFd, FromRawFd}; +use std::os::unix::ffi::{OsStrExt, OsStringExt}; +use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; + +use blaze_core::config::RuntimeTemplateSection; +use serde_json::json; +use tokio::sync::watch; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +use crate::error::{BlazeDaemonError, Result}; + +use super::manager::SandboxManager; + +const CATALOG_DIR_MODE: u32 = 0o700; +const CATALOG_FILE_MODE: u32 = 0o600; +const COPY_BUFFER_BYTES: usize = 1024 * 1024; + +#[derive(Clone, Copy)] +struct ImportLimits { + max_files: usize, + max_bytes: u64, + max_metadata_bytes: u64, + max_total_bytes: u64, +} + +#[derive(Clone)] +pub(crate) struct RuntimeTemplateCatalog { + inner: Arc, +} + +struct CatalogInner { + root: PathBuf, + import_root: Option, + limits: ImportLimits, + state: Mutex, + active_count: watch::Sender, + cancellation: CancellationToken, + #[cfg(test)] + copy_gate: Mutex>>, +} + +struct CatalogState { + active_names: HashSet, + committed_bytes: u64, + reserved_bytes: u64, + stopping: bool, + blocked: Option, +} + +struct ImportClaim { + inner: Arc, + name: String, + reserved_bytes: u64, +} + +struct PreparedFile { + name: OsString, + file: File, + observed_bytes: u64, + observed_dev: u64, + observed_ino: u64, + observed_mtime: i64, + observed_mtime_nsec: i64, + observed_ctime: i64, + observed_ctime_nsec: i64, +} + +struct PreparedImport { + files: Vec, + metadata: serde_json::Value, + metadata_bytes: Vec, + reserved_bytes: u64, +} + +#[cfg(test)] +struct TestCopyGate { + entered: tokio::sync::mpsc::UnboundedSender<()>, + release: AtomicBool, +} + +impl RuntimeTemplateCatalog { + pub(crate) fn open(config: &RuntimeTemplateSection) -> Result { + create_catalog_root(&config.dir)?; + cleanup_staging(&config.dir)?; + let limits = ImportLimits { + max_files: config.max_files, + max_bytes: config.max_bytes, + max_metadata_bytes: config.max_metadata_bytes, + max_total_bytes: config.max_total_bytes, + }; + let committed_bytes = catalog_usage(&config.dir, limits)?; + if committed_bytes > limits.max_total_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog uses {committed_bytes} bytes; configured limit is {}", + limits.max_total_bytes + ))); + } + let (active_count, _) = watch::channel(0); + Ok(Self { + inner: Arc::new(CatalogInner { + root: config.dir.clone(), + import_root: config.import_root.clone(), + limits, + state: Mutex::new(CatalogState { + active_names: HashSet::new(), + committed_bytes, + reserved_bytes: 0, + stopping: false, + blocked: None, + }), + active_count, + cancellation: CancellationToken::new(), + #[cfg(test)] + copy_gate: Mutex::new(None), + }), + }) + } + + async fn list(&self) -> Result> { + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + list_published(&catalog.inner.root, catalog.inner.limits) + }) + .await + .map_err(join_error("runtime template list"))? + } + + async fn get(&self, name: String) -> Result { + validate_name(&name, "runtime template")?; + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + get_published(&catalog.inner.root, &name, catalog.inner.limits) + }) + .await + .map_err(join_error("runtime template read"))? + } + + async fn import( + &self, + name: String, + source: PathBuf, + description: String, + ) -> Result { + validate_name(&name, "runtime template")?; + validate_relative_source(&source)?; + if self.inner.import_root.is_none() { + return Err(BlazeDaemonError::Conflict( + "runtime template import is disabled; configure \ + runtime_templates.import_root" + .to_string(), + )); + } + + // Register before scheduling blocking work. Shutdown can therefore + // observe and wait for an import even when the blocking pool has not + // started its closure yet. + let claim = ImportClaim::begin(Arc::clone(&self.inner), name.clone())?; + let catalog = self.clone(); + tokio::task::spawn_blocking(move || { + catalog.import_blocking(claim, name, source, description) + }) + .await + .map_err(join_error("runtime template import"))? + } + + fn import_blocking( + &self, + mut claim: ImportClaim, + name: String, + source: PathBuf, + description: String, + ) -> Result { + check_cancelled(&self.inner.cancellation)?; + let import_root = + self.inner.import_root.as_deref().ok_or_else(|| { + BlazeDaemonError::Conflict("runtime template import disabled".into()) + })?; + let source = open_import_source(import_root, &source)?; + let prepared = prepare_import( + &source, + &name, + &description, + self.inner.limits, + &self.inner.cancellation, + )?; + claim.reserve(prepared.reserved_bytes)?; + publish_prepared( + &self.inner.root, + &name, + prepared, + &self.inner.cancellation, + &mut claim, + ) + } + + pub(super) fn cancel_imports(&self) { + let mut state = lock_catalog_state(&self.inner); + state.stopping = true; + drop(state); + self.inner.cancellation.cancel(); + } + + pub(super) async fn wait_for_imports(&self) -> Result<()> { + let mut active = self.inner.active_count.subscribe(); + loop { + if *active.borrow_and_update() == 0 { + return Ok(()); + } + active.changed().await.map_err(|_| { + BlazeDaemonError::Internal( + "runtime template import supervisor closed unexpectedly".to_string(), + ) + })?; + } + } + + #[cfg(test)] + fn active_imports(&self) -> usize { + *self.inner.active_count.borrow() + } + + #[cfg(test)] + fn install_copy_gate(&self) -> tokio::sync::mpsc::UnboundedReceiver<()> { + let (entered, receiver) = tokio::sync::mpsc::unbounded_channel(); + let gate = Arc::new(TestCopyGate { + entered, + release: AtomicBool::new(false), + }); + *self.inner.copy_gate.lock().expect("copy gate lock") = Some(gate); + receiver + } +} + +impl ImportClaim { + fn begin(inner: Arc, name: String) -> Result { + let mut state = lock_catalog_state(&inner); + if state.stopping { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template imports are stopping".to_string(), + )); + } + if let Some(error) = &state.blocked { + return Err(BlazeDaemonError::RecoveryRequired(error.clone())); + } + if !state.active_names.insert(name.clone()) { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} import is already in progress" + ))); + } + let count = state.active_names.len(); + inner.active_count.send_replace(count); + drop(state); + Ok(Self { + inner, + name, + reserved_bytes: 0, + }) + } + + fn reserve(&mut self, bytes: u64) -> Result<()> { + let mut state = lock_catalog_state(&self.inner); + if state.stopping || self.inner.cancellation.is_cancelled() { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template imports are stopping".to_string(), + )); + } + if let Some(error) = &state.blocked { + return Err(BlazeDaemonError::RecoveryRequired(error.clone())); + } + let used = state + .committed_bytes + .checked_add(state.reserved_bytes) + .and_then(|value| value.checked_add(bytes)) + .ok_or_else(|| payload_too_large(u64::MAX, self.inner.limits.max_total_bytes))?; + if used > self.inner.limits.max_total_bytes { + return Err(payload_too_large(used, self.inner.limits.max_total_bytes)); + } + state.reserved_bytes += bytes; + self.reserved_bytes = bytes; + Ok(()) + } + + fn publish(&mut self, actual_bytes: u64) -> Result<()> { + if actual_bytes > self.reserved_bytes { + let message = format!( + "runtime template {} wrote {actual_bytes} bytes beyond its {}-byte reservation", + self.name, self.reserved_bytes + ); + self.block_catalog(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + let mut state = lock_catalog_state(&self.inner); + let Some(remaining_reserved) = state.reserved_bytes.checked_sub(self.reserved_bytes) else { + let message = "runtime template reservation accounting underflow".to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + }; + let Some(committed_bytes) = state.committed_bytes.checked_add(actual_bytes) else { + let message = "runtime template catalog byte accounting overflow".to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + }; + if committed_bytes + .checked_add(remaining_reserved) + .is_none_or(|used| used > self.inner.limits.max_total_bytes) + { + let message = "runtime template catalog accounting exceeded the configured total limit" + .to_string(); + state.blocked = Some(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + state.reserved_bytes = remaining_reserved; + state.committed_bytes = committed_bytes; + self.reserved_bytes = 0; + Ok(()) + } + + fn block_catalog(&self, message: String) { + let mut state = lock_catalog_state(&self.inner); + state.blocked = Some(message); + } +} + +impl Drop for ImportClaim { + fn drop(&mut self) { + let mut state = lock_catalog_state(&self.inner); + if self.reserved_bytes > 0 { + state.reserved_bytes = state.reserved_bytes.saturating_sub(self.reserved_bytes); + } + state.active_names.remove(&self.name); + let count = state.active_names.len(); + self.inner.active_count.send_replace(count); + } +} + +impl SandboxManager { + /// List atomically published runtime artifact sets. + pub async fn list_runtime_templates(&self) -> Result> { + self.runtime_templates.list().await + } + + /// Read one published runtime artifact set by name. + pub async fn get_runtime_template(&self, name: String) -> Result { + self.runtime_templates.get(name).await + } + + /// Copy and atomically publish one operator-prepared artifact directory. + pub async fn import_runtime_template( + &self, + name: String, + source: PathBuf, + description: String, + ) -> Result { + self.runtime_templates + .import(name, source, description) + .await + } + + /// Reject new imports and request cancellation of every active import. + pub(crate) fn cancel_runtime_template_imports(&self) { + self.runtime_templates.cancel_imports(); + } + + /// Wait until every registered import has released its filesystem handles. + pub(crate) async fn wait_for_runtime_template_imports(&self) -> Result<()> { + self.runtime_templates.wait_for_imports().await + } +} + +fn prepare_import( + source: &File, + name: &str, + description: &str, + limits: ImportLimits, + cancellation: &CancellationToken, +) -> Result { + let names = source_entry_names(source)?; + let mut files = Vec::with_capacity(names.len()); + let mut metadata_file = None; + let mut artifact_bytes = 0_u64; + + for entry_name in names { + check_cancelled(cancellation)?; + validate_artifact_name(&entry_name)?; + let file = openat_regular(source, &entry_name)?; + let metadata = file.metadata()?; + validate_source_file(&metadata, &entry_name)?; + if entry_name == OsStr::new("template.json") { + if metadata.len() > limits.max_metadata_bytes { + return Err(payload_too_large(metadata.len(), limits.max_metadata_bytes)); + } + metadata_file = Some(file); + } else { + artifact_bytes = artifact_bytes + .checked_add(metadata.len()) + .ok_or_else(|| payload_too_large(u64::MAX, limits.max_bytes))?; + files.push(PreparedFile { + name: entry_name, + file, + observed_bytes: metadata.len(), + observed_dev: metadata.dev(), + observed_ino: metadata.ino(), + observed_mtime: metadata.mtime(), + observed_mtime_nsec: metadata.mtime_nsec(), + observed_ctime: metadata.ctime(), + observed_ctime_nsec: metadata.ctime_nsec(), + }); + } + } + + let published_files = files.len() + 1; + if published_files > limits.max_files { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template contains {published_files} files; limit is {}", + limits.max_files + ))); + } + let present = files + .iter() + .map(|file| file.name.as_os_str()) + .collect::>(); + for required in ["vmstate.snap", "mem.bin", "rootfs.ext4"] { + if !present.contains(OsStr::new(required)) { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source is missing regular artifact {required}" + ))); + } + } + + let mut metadata = match metadata_file { + Some(mut file) => { + let observed = file.metadata()?; + let metadata = read_json_bounded(&mut file, limits.max_metadata_bytes)?; + let current = file.metadata()?; + if !same_file_identity(&observed, ¤t) { + return Err(BlazeDaemonError::BadRequest( + "runtime template source metadata changed while it was imported".to_string(), + )); + } + metadata + } + None => json!({"name": name}), + }; + if !metadata.is_object() { + return Err(BlazeDaemonError::BadRequest( + "template.json must contain a JSON object".to_string(), + )); + } + metadata["name"] = json!(name); + if !description.is_empty() { + metadata["description"] = json!(description); + } + if metadata + .get("rootfs_size") + .and_then(serde_json::Value::as_u64) + .is_none() + { + metadata["rootfs_size"] = json!(8_u64 * 1024 * 1024 * 1024); + } + if metadata + .get("memory_size") + .and_then(serde_json::Value::as_u64) + .is_none() + { + metadata["memory_size"] = json!(4_u64 * 1024 * 1024 * 1024); + } + let metadata_bytes = serde_json::to_vec_pretty(&metadata)?; + let metadata_len = u64::try_from(metadata_bytes.len()).unwrap_or(u64::MAX); + if metadata_len > limits.max_metadata_bytes { + return Err(payload_too_large(metadata_len, limits.max_metadata_bytes)); + } + let reserved_bytes = artifact_bytes + .checked_add(metadata_len) + .ok_or_else(|| payload_too_large(u64::MAX, limits.max_bytes))?; + if reserved_bytes > limits.max_bytes { + return Err(payload_too_large(reserved_bytes, limits.max_bytes)); + } + + Ok(PreparedImport { + files, + metadata, + metadata_bytes, + reserved_bytes, + }) +} + +fn publish_prepared( + root: &Path, + name: &str, + prepared: PreparedImport, + cancellation: &CancellationToken, + claim: &mut ImportClaim, +) -> Result { + let destination = root.join(name); + if destination.exists() { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} already exists" + ))); + } + + let staging = root.join(format!(".import-{name}-{}.tmp", Uuid::new_v4())); + create_private_directory(&staging)?; + #[cfg(test)] + wait_for_copy_gate(&claim.inner, cancellation); + let result = populate_and_publish( + root, + &destination, + &staging, + name, + prepared, + cancellation, + claim, + ); + if result.is_err() && staging.exists() { + if let Err(cleanup_error) = std::fs::remove_dir_all(&staging) { + claim.block_catalog(format!( + "runtime template staging cleanup failed; restart after repairing the catalog: \ + {cleanup_error}" + )); + tracing::error!( + path = %staging.display(), + error = %cleanup_error, + "runtime template staging cleanup failed" + ); + } else if let Err(sync_error) = sync_directory(root) { + claim.block_catalog(format!( + "runtime template cleanup durability is unknown; restart after repairing the \ + catalog: {sync_error}" + )); + tracing::error!( + path = %root.display(), + error = %sync_error, + "runtime template cleanup durability is unknown" + ); + } + } + result +} + +fn populate_and_publish( + root: &Path, + destination: &Path, + staging: &Path, + name: &str, + mut prepared: PreparedImport, + cancellation: &CancellationToken, + claim: &mut ImportClaim, +) -> Result { + let mut actual_bytes = 0_u64; + for source in &mut prepared.files { + check_cancelled(cancellation)?; + let remaining = prepared + .reserved_bytes + .checked_sub(actual_bytes) + .and_then(|value| { + value.checked_sub(u64::try_from(prepared.metadata_bytes.len()).unwrap_or(u64::MAX)) + }) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + let destination_file = staging.join(&source.name); + let copied = + copy_regular_file(&mut source.file, &destination_file, remaining, cancellation)?; + let current = source.file.metadata()?; + if copied != source.observed_bytes + || current.len() != source.observed_bytes + || current.dev() != source.observed_dev + || current.ino() != source.observed_ino + || current.mtime() != source.observed_mtime + || current.mtime_nsec() != source.observed_mtime_nsec + || current.ctime() != source.observed_ctime + || current.ctime_nsec() != source.observed_ctime_nsec + { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source file {} changed while it was imported", + source.name.to_string_lossy() + ))); + } + actual_bytes = actual_bytes + .checked_add(copied) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + } + + let metadata_len = u64::try_from(prepared.metadata_bytes.len()).unwrap_or(u64::MAX); + actual_bytes = actual_bytes + .checked_add(metadata_len) + .ok_or_else(|| payload_too_large(u64::MAX, prepared.reserved_bytes))?; + if actual_bytes > prepared.reserved_bytes { + return Err(payload_too_large(actual_bytes, prepared.reserved_bytes)); + } + write_file_durable(&staging.join("template.json"), &prepared.metadata_bytes)?; + sync_directory(staging)?; + check_cancelled(cancellation)?; + + if destination.exists() { + return Err(BlazeDaemonError::Conflict(format!( + "runtime template {name} already exists" + ))); + } + rename_no_replace(staging, destination).map_err(|error| { + if destination.exists() { + BlazeDaemonError::Conflict(format!("runtime template {name} already exists")) + } else { + error.into() + } + })?; + + // The directory is now publicly owned even if the parent fsync fails. + // Account for it before reporting an uncertain durability result. + claim.publish(actual_bytes)?; + if let Err(error) = sync_directory(root) { + let message = format!( + "runtime template {name} was published but catalog durability is unknown: {error}" + ); + claim.block_catalog(message.clone()); + return Err(BlazeDaemonError::RecoveryRequired(message)); + } + Ok(prepared.metadata) +} + +fn open_import_source(import_root: &Path, relative: &Path) -> Result { + let mut directory = open_directory_no_follow(import_root).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open configured runtime template import root {}: {error}", + import_root.display() + )) + })?; + validate_source_directory(&directory.metadata()?, import_root)?; + for component in relative.components() { + let Component::Normal(name) = component else { + return Err(BlazeDaemonError::BadRequest( + "runtime template source must be a non-empty relative path below the configured \ + import root" + .to_string(), + )); + }; + directory = openat_directory(&directory, name).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open runtime template source {}: {error}", + relative.display() + )) + })?; + validate_source_directory(&directory.metadata()?, relative)?; + } + Ok(directory) +} + +fn source_entry_names(directory: &File) -> Result> { + let duplicated = unsafe { libc::dup(directory.as_raw_fd()) }; + if duplicated < 0 { + return Err(io::Error::last_os_error().into()); + } + let stream = unsafe { libc::fdopendir(duplicated) }; + if stream.is_null() { + let error = io::Error::last_os_error(); + unsafe { + libc::close(duplicated); + } + return Err(error.into()); + } + let mut names = Vec::new(); + loop { + clear_errno(); + let entry = unsafe { libc::readdir(stream) }; + if entry.is_null() { + let error = io::Error::last_os_error(); + if error.raw_os_error() != Some(0) { + unsafe { + libc::closedir(stream); + } + return Err(error.into()); + } + break; + } + let name = unsafe { CStr::from_ptr((*entry).d_name.as_ptr()) }.to_bytes(); + if name != b"." && name != b".." { + names.push(OsString::from_vec(name.to_vec())); + } + } + let close_result = unsafe { libc::closedir(stream) }; + if close_result != 0 { + return Err(io::Error::last_os_error().into()); + } + names.sort(); + Ok(names) +} + +#[cfg(target_os = "linux")] +fn clear_errno() { + unsafe { + *libc::__errno_location() = 0; + } +} + +#[cfg(not(target_os = "linux"))] +fn clear_errno() { + unsafe { + *libc::__error() = 0; + } +} + +fn open_directory_no_follow(path: &Path) -> io::Result { + let path = CString::new(path.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "path contains NUL"))?; + let fd = unsafe { + libc::open( + path.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW, + ) + }; + file_from_fd(fd) +} + +fn openat_directory(parent: &File, name: &OsStr) -> io::Result { + let name = CString::new(name.as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "name contains NUL"))?; + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_DIRECTORY | libc::O_NOFOLLOW, + ) + }; + file_from_fd(fd) +} + +fn openat_regular(parent: &File, name: &OsStr) -> Result { + let name_c = CString::new(name.as_bytes()) + .map_err(|_| BlazeDaemonError::BadRequest("artifact name contains NUL".to_string()))?; + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name_c.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK, + ) + }; + file_from_fd(fd).map_err(|error| { + BlazeDaemonError::BadRequest(format!( + "cannot open runtime template source entry {} without following links: {error}", + name.to_string_lossy() + )) + }) +} + +fn file_from_fd(fd: libc::c_int) -> io::Result { + if fd < 0 { + return Err(io::Error::last_os_error()); + } + Ok(unsafe { File::from_raw_fd(fd) }) +} + +fn validate_source_directory(metadata: &std::fs::Metadata, path: &Path) -> Result<()> { + if !metadata.is_dir() { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source {} is not a directory", + path.display() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid || metadata.mode() & 0o022 != 0 { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source directory {} must be owned by the daemon user and not \ + writable by group or other users", + path.display() + ))); + } + Ok(()) +} + +fn validate_source_file(metadata: &std::fs::Metadata, name: &OsStr) -> Result<()> { + if !metadata.file_type().is_file() { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source entry {} is not a regular file", + name.to_string_lossy() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid || metadata.mode() & 0o022 != 0 { + return Err(BlazeDaemonError::BadRequest(format!( + "runtime template source file {} must be owned by the daemon user and not writable \ + by group or other users", + name.to_string_lossy() + ))); + } + Ok(()) +} + +fn same_file_identity(observed: &std::fs::Metadata, current: &std::fs::Metadata) -> bool { + observed.len() == current.len() + && observed.dev() == current.dev() + && observed.ino() == current.ino() + && observed.mtime() == current.mtime() + && observed.mtime_nsec() == current.mtime_nsec() + && observed.ctime() == current.ctime() + && observed.ctime_nsec() == current.ctime_nsec() +} + +fn create_catalog_root(root: &Path) -> Result<()> { + if !root.exists() { + DirBuilder::new() + .recursive(true) + .mode(CATALOG_DIR_MODE) + .create(root)?; + if let Some(parent) = root.parent() { + sync_directory(parent)?; + } + } + enforce_owned_mode(root, true, CATALOG_DIR_MODE) +} + +fn create_private_directory(path: &Path) -> Result<()> { + DirBuilder::new().mode(CATALOG_DIR_MODE).create(path)?; + enforce_owned_mode(path, true, CATALOG_DIR_MODE) +} + +fn enforce_owned_mode(path: &Path, directory: bool, mode: u32) -> Result<()> { + let metadata = std::fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() + || (directory && !metadata.is_dir()) + || (!directory && !metadata.file_type().is_file()) + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog path {} has an unexpected file type", + path.display() + ))); + } + let expected_uid = unsafe { libc::geteuid() }; + if metadata.uid() != expected_uid { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog path {} is not owned by the daemon user", + path.display() + ))); + } + if metadata.mode() & 0o777 != mode { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode))?; + } + Ok(()) +} + +fn catalog_usage(root: &Path, limits: ImportLimits) -> Result { + let mut total = 0_u64; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if name.to_string_lossy().starts_with('.') { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains unresolved hidden entry {}", + entry.path().display() + ))); + } + enforce_owned_mode(&entry.path(), true, CATALOG_DIR_MODE)?; + let mut file_count = 0_usize; + let mut template_bytes = 0_u64; + for artifact in std::fs::read_dir(entry.path())? { + let artifact = artifact?; + enforce_owned_mode(&artifact.path(), false, CATALOG_FILE_MODE)?; + file_count = file_count.checked_add(1).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog file count overflow".to_string(), + ) + })?; + if file_count > limits.max_files { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} exceeds the configured file limit", + name.to_string_lossy() + ))); + } + let artifact_bytes = artifact.metadata()?.len(); + if artifact.file_name() == OsStr::new("template.json") + && artifact_bytes > limits.max_metadata_bytes + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} metadata exceeds the configured limit", + name.to_string_lossy() + ))); + } + template_bytes = template_bytes.checked_add(artifact_bytes).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template byte accounting overflow".to_string(), + ) + })?; + total = total.checked_add(artifact_bytes).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog byte accounting overflow".to_string(), + ) + })?; + } + if template_bytes > limits.max_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {} exceeds the configured per-import byte limit", + name.to_string_lossy() + ))); + } + let name = name.into_string().map_err(|_| { + BlazeDaemonError::RecoveryRequired( + "runtime template catalog contains a non-UTF-8 published name".to_string(), + ) + })?; + validate_name(&name, "runtime template").map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains invalid published name {name}: {error}" + )) + })?; + read_published(root, &name, limits)?; + } + Ok(total) +} + +fn list_published(root: &Path, limits: ImportLimits) -> Result> { + let mut templates = Vec::new(); + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if name.to_string_lossy().starts_with('.') { + continue; + } + let name = name.into_string().map_err(|_| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template catalog contains a non-UTF-8 name at {}", + entry.path().display() + )) + })?; + templates.push(read_published(root, &name, limits)?); + } + templates.sort_by(|left, right| { + left.get("name") + .and_then(serde_json::Value::as_str) + .cmp(&right.get("name").and_then(serde_json::Value::as_str)) + }); + Ok(templates) +} + +fn get_published(root: &Path, name: &str, limits: ImportLimits) -> Result { + let root = open_directory_no_follow(root).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot open runtime template catalog {}: {error}", + root.display() + )) + })?; + let directory = match openat_directory(&root, OsStr::new(name)) { + Ok(directory) => directory, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + return Err(BlazeDaemonError::NotFound(format!( + "runtime template {name}" + ))); + } + Err(error) => return Err(error.into()), + }; + read_published_directory(&directory, name, limits) +} + +fn read_published(root: &Path, name: &str, limits: ImportLimits) -> Result { + let root = open_directory_no_follow(root)?; + let directory = openat_directory(&root, OsStr::new(name)).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!("cannot open runtime template {name}: {error}")) + })?; + read_published_directory(&directory, name, limits) +} + +fn read_published_directory( + directory: &File, + expected_name: &str, + limits: ImportLimits, +) -> Result { + let names = source_entry_names(directory).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot inspect runtime template {expected_name}: {error}" + )) + })?; + if names.len() > limits.max_files { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} exceeds the configured file limit" + ))); + } + let mut total_bytes = 0_u64; + for name in names { + validate_artifact_name(&name).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} contains an invalid artifact: {error}" + )) + })?; + let file = openat_regular(directory, &name).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} contains an invalid artifact {}: {error}", + name.to_string_lossy() + )) + })?; + let metadata = file.metadata()?; + validate_published_file(&metadata, expected_name, &name)?; + if name == OsStr::new("template.json") && metadata.len() > limits.max_metadata_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} metadata exceeds the configured limit" + ))); + } + total_bytes = total_bytes.checked_add(metadata.len()).ok_or_else(|| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} byte accounting overflow" + )) + })?; + } + if total_bytes > limits.max_bytes { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} exceeds the configured byte limit" + ))); + } + + let mut metadata = openat_regular(directory, OsStr::new("template.json")).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot open runtime template {expected_name} metadata: {error}" + )) + })?; + let value = read_json_bounded(&mut metadata, limits.max_metadata_bytes).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "cannot read runtime template {expected_name} metadata: {error}" + )) + })?; + if !value.is_object() + || value.get("name").and_then(serde_json::Value::as_str) != Some(expected_name) + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} metadata does not match its catalog name" + ))); + } + for required in ["vmstate.snap", "mem.bin", "rootfs.ext4"] { + openat_regular(directory, OsStr::new(required)).map_err(|error| { + BlazeDaemonError::RecoveryRequired(format!( + "runtime template {expected_name} is missing regular artifact {required}: {error}" + )) + })?; + } + Ok(value) +} + +fn validate_published_file( + metadata: &std::fs::Metadata, + template: &str, + name: &OsStr, +) -> Result<()> { + let expected_uid = unsafe { libc::geteuid() }; + if !metadata.file_type().is_file() + || metadata.uid() != expected_uid + || metadata.mode() & 0o777 != CATALOG_FILE_MODE + { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template {template} artifact {} has unexpected type, ownership, or mode", + name.to_string_lossy() + ))); + } + Ok(()) +} + +fn copy_regular_file( + source: &mut File, + destination: &Path, + max_bytes: u64, + cancellation: &CancellationToken, +) -> Result { + let mut destination = OpenOptions::new() + .write(true) + .create_new(true) + .mode(CATALOG_FILE_MODE) + .custom_flags(libc::O_NOFOLLOW) + .open(destination)?; + let mut buffer = vec![0_u8; COPY_BUFFER_BYTES]; + let mut copied = 0_u64; + loop { + check_cancelled(cancellation)?; + let read = source.read(&mut buffer)?; + if read == 0 { + break; + } + let next = copied + .checked_add(u64::try_from(read).unwrap_or(u64::MAX)) + .ok_or_else(|| payload_too_large(u64::MAX, max_bytes))?; + if next > max_bytes { + return Err(payload_too_large(next, max_bytes)); + } + destination.write_all(&buffer[..read])?; + copied = next; + } + destination.sync_all()?; + destination.set_permissions(std::fs::Permissions::from_mode(CATALOG_FILE_MODE))?; + Ok(copied) +} + +fn write_file_durable(path: &Path, bytes: &[u8]) -> Result<()> { + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(CATALOG_FILE_MODE) + .custom_flags(libc::O_NOFOLLOW) + .open(path)?; + file.write_all(bytes)?; + file.sync_all()?; + file.set_permissions(std::fs::Permissions::from_mode(CATALOG_FILE_MODE))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn rename_no_replace(source: &Path, destination: &Path) -> io::Result<()> { + rename_no_replace_linux(source, destination) +} + +#[cfg(not(target_os = "linux"))] +fn rename_no_replace(source: &Path, destination: &Path) -> io::Result<()> { + if destination.exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + "destination already exists", + )); + } + std::fs::rename(source, destination) +} + +#[cfg(target_os = "linux")] +fn rename_no_replace_linux(source: &Path, destination: &Path) -> io::Result<()> { + let source = CString::new(source.as_os_str().as_bytes()) + .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "source path contains NUL"))?; + let destination = CString::new(destination.as_os_str().as_bytes()).map_err(|_| { + io::Error::new(io::ErrorKind::InvalidInput, "destination path contains NUL") + })?; + let result = unsafe { + libc::renameat2( + libc::AT_FDCWD, + source.as_ptr(), + libc::AT_FDCWD, + destination.as_ptr(), + libc::RENAME_NOREPLACE, + ) + }; + if result == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +fn read_json_bounded(file: &mut File, limit: u64) -> Result { + let mut bytes = Vec::new(); + file.take(limit.saturating_add(1)).read_to_end(&mut bytes)?; + let actual = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + if actual > limit { + return Err(payload_too_large(actual, limit)); + } + Ok(serde_json::from_slice(&bytes)?) +} + +fn sync_directory(path: &Path) -> Result<()> { + open_directory_no_follow(path)?.sync_all()?; + Ok(()) +} + +fn cleanup_staging(root: &Path) -> Result { + let mut removed = 0; + for entry in std::fs::read_dir(root)? { + let entry = entry?; + let name = entry.file_name(); + if !is_staging_name(&name) { + continue; + } + let metadata = std::fs::symlink_metadata(entry.path())?; + let expected_uid = unsafe { libc::geteuid() }; + if !metadata.is_dir() || metadata.uid() != expected_uid { + return Err(BlazeDaemonError::RecoveryRequired(format!( + "runtime template staging entry {} has unexpected ownership or type", + entry.path().display() + ))); + } + std::fs::remove_dir_all(entry.path())?; + removed += 1; + } + if removed > 0 { + sync_directory(root)?; + tracing::info!( + removed, + "removed stale runtime template staging directories" + ); + } + Ok(removed) +} + +fn is_staging_name(name: &OsStr) -> bool { + let name = name.to_string_lossy(); + name.starts_with(".import-") && name.ends_with(".tmp") +} + +fn validate_name(value: &str, label: &str) -> Result<()> { + let mut chars = value.chars(); + let first = chars.next(); + if value.len() > 128 + || !first.is_some_and(|ch| ch.is_ascii_alphanumeric()) + || !chars.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + { + return Err(BlazeDaemonError::BadRequest(format!( + "{label} must start with an ASCII letter or digit and contain at most 128 \ + letters, digits, dots, dashes, or underscores" + ))); + } + Ok(()) +} + +fn validate_artifact_name(value: &OsStr) -> Result<()> { + let value = value.to_str().ok_or_else(|| { + BlazeDaemonError::BadRequest( + "runtime template artifact names must be valid UTF-8".to_string(), + ) + })?; + validate_name(value, "runtime template artifact") +} + +fn validate_relative_source(source: &Path) -> Result<()> { + if source.as_os_str().is_empty() + || source + .components() + .any(|component| !matches!(component, Component::Normal(_))) + { + return Err(BlazeDaemonError::BadRequest( + "runtime template source must be a non-empty relative path below the configured \ + import root" + .to_string(), + )); + } + Ok(()) +} + +fn check_cancelled(cancellation: &CancellationToken) -> Result<()> { + if cancellation.is_cancelled() { + return Err(BlazeDaemonError::ServiceUnavailable( + "runtime template import cancelled during daemon shutdown".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +fn wait_for_copy_gate(inner: &CatalogInner, cancellation: &CancellationToken) { + let gate = inner.copy_gate.lock().expect("copy gate lock").clone(); + if let Some(gate) = gate { + let _ = gate.entered.send(()); + while !gate.release.load(Ordering::Acquire) && !cancellation.is_cancelled() { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } +} + +fn payload_too_large(actual: u64, limit: u64) -> BlazeDaemonError { + BlazeDaemonError::PayloadTooLarge { + actual, + limit: usize::try_from(limit).unwrap_or(usize::MAX), + } +} + +fn join_error(context: &'static str) -> impl FnOnce(tokio::task::JoinError) -> BlazeDaemonError { + move |error| BlazeDaemonError::Internal(format!("{context} task: {error}")) +} + +fn lock_catalog_state(inner: &CatalogInner) -> std::sync::MutexGuard<'_, CatalogState> { + match inner.state.lock() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn test_config(root: &Path, import_root: &Path) -> RuntimeTemplateSection { + RuntimeTemplateSection { + dir: root.to_path_buf(), + import_root: Some(import_root.to_path_buf()), + max_files: 8, + max_bytes: 1024, + max_metadata_bytes: 512, + max_total_bytes: 2048, + } + } + + #[tokio::test] + async fn import_publishes_artifacts_with_private_permissions() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + let metadata = catalog + .import( + "runtime-base".to_string(), + PathBuf::from("source"), + "base runtime template".to_string(), + ) + .await + .expect("import"); + let destination = root.join("runtime-base"); + + assert_eq!(metadata["name"], "runtime-base"); + assert_eq!(metadata["description"], "base runtime template"); + assert_eq!( + std::fs::symlink_metadata(&destination) + .expect("directory") + .mode() + & 0o777, + CATALOG_DIR_MODE + ); + for file in ["vmstate.snap", "mem.bin", "rootfs.ext4", "template.json"] { + assert_eq!( + std::fs::symlink_metadata(destination.join(file)) + .expect("artifact") + .mode() + & 0o777, + CATALOG_FILE_MODE + ); + } + } + + #[tokio::test] + async fn import_rejects_special_entries_and_cleans_staging() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let fifo = CString::new(source.join("fifo").as_os_str().as_bytes()).expect("fifo path"); + assert_eq!(unsafe { libc::mkfifo(fifo.as_ptr(), 0o600) }, 0); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + catalog + .import("special".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("special file"); + + assert!(!root.join("special").exists()); + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[tokio::test] + async fn import_does_not_follow_source_links() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + symlink("mem.bin", source.join("linked-memory")).expect("source link"); + symlink("source", import_root.join("source-link")).expect("directory link"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + catalog + .import("file-link".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("file link"); + catalog + .import( + "directory-link".into(), + PathBuf::from("source-link"), + String::new(), + ) + .await + .expect_err("directory link"); + + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[tokio::test] + async fn metadata_and_catalog_capacity_are_enforced() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let mut config = test_config(&root, &import_root); + config.max_metadata_bytes = 64; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + + let error = catalog + .import("metadata".into(), PathBuf::from("source"), "x".repeat(128)) + .await + .expect_err("metadata limit"); + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + + let mut config = test_config(&root, &import_root); + config.max_total_bytes = 8; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + let error = catalog + .import("capacity".into(), PathBuf::from("source"), String::new()) + .await + .expect_err("catalog capacity"); + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + } + + #[test] + fn concurrent_reservations_share_one_catalog_limit() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let mut config = test_config(&root, &import_root); + config.max_total_bytes = 100; + let catalog = RuntimeTemplateCatalog::open(&config).expect("catalog"); + let mut first = + ImportClaim::begin(Arc::clone(&catalog.inner), "first".into()).expect("first claim"); + let mut second = + ImportClaim::begin(Arc::clone(&catalog.inner), "second".into()).expect("second claim"); + + first.reserve(60).expect("first reservation"); + assert!(matches!( + second.reserve(60), + Err(BlazeDaemonError::PayloadTooLarge { .. }) + )); + } + + #[test] + fn accounting_failure_blocks_later_imports() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let mut claim = + ImportClaim::begin(Arc::clone(&catalog.inner), "first".into()).expect("claim"); + claim.reserve(10).expect("reservation"); + + let error = claim.publish(11).expect_err("reservation mismatch"); + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + assert!(matches!( + ImportClaim::begin(Arc::clone(&catalog.inner), "later".into()), + Err(BlazeDaemonError::RecoveryRequired(_)) + )); + } + + #[test] + fn copy_counts_bytes_read_after_preflight() { + let temp = tempfile::tempdir().expect("tempdir"); + let source_path = temp.path().join("source"); + std::fs::write(&source_path, b"one").expect("source"); + let mut source = OpenOptions::new() + .read(true) + .open(&source_path) + .expect("open source"); + std::fs::write(&source_path, b"longer").expect("grow source"); + let destination = temp.path().join("destination"); + + let error = copy_regular_file(&mut source, &destination, 3, &CancellationToken::new()) + .expect_err("actual bytes exceed reservation"); + + assert!(matches!(error, BlazeDaemonError::PayloadTooLarge { .. })); + } + + #[tokio::test] + async fn shutdown_waits_for_registered_import_claims() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let claim = ImportClaim::begin(Arc::clone(&catalog.inner), "active".into()).expect("claim"); + assert_eq!(catalog.active_imports(), 1); + + catalog.cancel_imports(); + let waiting = tokio::spawn({ + let catalog = catalog.clone(); + async move { catalog.wait_for_imports().await } + }); + tokio::task::yield_now().await; + assert!(!waiting.is_finished()); + + drop(claim); + waiting.await.expect("join").expect("imports stopped"); + assert_eq!(catalog.active_imports(), 0); + assert!(matches!( + ImportClaim::begin(Arc::clone(&catalog.inner), "late".into()), + Err(BlazeDaemonError::ServiceUnavailable(_)) + )); + } + + #[tokio::test] + async fn shutdown_cancels_copy_and_removes_staging() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + let source = import_root.join("source"); + let root = temp.path().join("catalog"); + write_artifacts(&source); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let mut entered = catalog.install_copy_gate(); + let import = tokio::spawn({ + let catalog = catalog.clone(); + async move { + catalog + .import("cancelled".into(), PathBuf::from("source"), String::new()) + .await + } + }); + entered.recv().await.expect("copy entered"); + + catalog.cancel_imports(); + catalog.wait_for_imports().await.expect("imports quiescent"); + let error = import + .await + .expect("import task") + .expect_err("cancelled import"); + + assert!(matches!(error, BlazeDaemonError::ServiceUnavailable(_))); + assert!(!root.join("cancelled").exists()); + assert_eq!(std::fs::read_dir(root).expect("catalog").count(), 0); + } + + #[test] + fn list_reports_corrupt_published_metadata() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + let catalog = + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + let published = root.join("published"); + create_private_directory(&published).expect("published"); + write_file_durable(&published.join("template.json"), b"{broken").expect("metadata"); + + let error = list_published(&catalog.inner.root, catalog.inner.limits) + .expect_err("corrupt metadata"); + assert!(matches!(error, BlazeDaemonError::RecoveryRequired(_))); + } + + #[test] + fn startup_removes_owned_staging_directories() { + let temp = tempfile::tempdir().expect("tempdir"); + let import_root = temp.path().join("imports"); + std::fs::create_dir(&import_root).expect("import root"); + let root = temp.path().join("catalog"); + create_catalog_root(&root).expect("root"); + let staging = root.join(".import-pending-uuid.tmp"); + create_private_directory(&staging).expect("staging"); + + RuntimeTemplateCatalog::open(&test_config(&root, &import_root)).expect("catalog"); + + assert!(!staging.exists()); + } + + fn write_artifacts(source: &Path) { + std::fs::create_dir_all(source).expect("source directory"); + std::fs::write(source.join("vmstate.snap"), b"snapshot").expect("snapshot"); + std::fs::write(source.join("mem.bin"), b"memory").expect("memory"); + std::fs::write(source.join("rootfs.ext4"), b"rootfs").expect("rootfs"); + } +} diff --git a/src/blaze/crates/blazed/src/state.rs b/src/blaze/crates/blazed/src/state.rs index be8b5cf5b1..d005fb2e7e 100644 --- a/src/blaze/crates/blazed/src/state.rs +++ b/src/blaze/crates/blazed/src/state.rs @@ -23,6 +23,7 @@ use uuid::Uuid; use crate::error::{BlazeDaemonError, Result}; use crate::metrics::Metrics; +use crate::sandbox::template::RuntimeTemplateCatalog; use crate::sandbox::{SandboxManager, SandboxManagerInit}; use crate::spawner::SpawnerRegistry; @@ -63,6 +64,7 @@ impl ServerState { ) -> Result { let state_dir = config.daemon.state_dir.clone(); let instances = scan_state_dir(&state_dir)?; + let runtime_templates = RuntimeTemplateCatalog::open(&config.runtime_templates)?; let (manager, resources) = SandboxManager::new(SandboxManagerInit { instances, pool, @@ -72,6 +74,7 @@ impl ServerState { state_dir: state_dir.clone(), rootfs_size: config.storage.rootfs_size, mem_size: config.storage.mem_size, + runtime_templates, }); Ok(Self { diff --git a/src/blaze/examples/config.toml b/src/blaze/examples/config.toml index 1b1bc127d7..e4a65c91c5 100644 --- a/src/blaze/examples/config.toml +++ b/src/blaze/examples/config.toml @@ -134,6 +134,23 @@ gc_interval = "10m" # Format: duration string. Default: "1h". idle_ttl = "1h" +# --------------------------------------------------------------------------- +# Published runtime artifact catalog. +# --------------------------------------------------------------------------- +[runtime_templates] +# Destination for atomically published runtime artifact sets. +dir = "/var/lib/blaze/runtime-templates" + +# Optional operator-controlled source root. Imports are disabled when omitted. +# API requests name a relative directory below this root. +# import_root = "/srv/blaze/runtime-template-imports" + +# Per-import resource bounds and aggregate catalog capacity. +max_files = 32 +max_bytes = 274877906944 +max_metadata_bytes = 1048576 +max_total_bytes = 1099511627776 + # --------------------------------------------------------------------------- # Metrics / observability. # --------------------------------------------------------------------------- From 638f74413215c5faa85fc9be238ae48c62326afd Mon Sep 17 00:00:00 2001 From: Weisson Date: Fri, 31 Jul 2026 17:36:55 +0800 Subject: [PATCH 38/40] docs(blaze): explain runtime artifact catalog This explains how operators configure and use the runtime artifact catalog through its import, list, and lookup endpoints. The guide documents source requirements, capacity limits, private output permissions, atomic publication, startup checks, and shutdown behavior. It also distinguishes this durable catalog from the existing in-memory template registry. Sandbox creation, reference counting, and catalog deletion remain out of scope. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/README.md | 13 ++- src/blaze/README_zh.md | 12 ++- .../docs/design/runtime-template-catalog.md | 88 +++++++++++++++++++ .../design/runtime-template-catalog_zh.md | 76 ++++++++++++++++ 4 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 src/blaze/docs/design/runtime-template-catalog.md create mode 100644 src/blaze/docs/design/runtime-template-catalog_zh.md diff --git a/src/blaze/README.md b/src/blaze/README.md index e8988d66f5..1f02d1d89a 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -128,14 +128,23 @@ The `file` provider uses standard filesystem operations for sandbox storage. The | GET | `/v1/pools/{backend}/{class}` | Get pool status | | POST | `/v1/pools/{backend}/{class}/drain` | Drain a pool | | PUT | `/v1/pools/{backend}/{class}/sizing` | Resize a pool | -| GET | `/v1/templates` | List templates | -| GET | `/v1/templates/{id}` | Inspect a template | +| GET | `/v1/templates` | List in-memory template registry entries | +| GET | `/v1/templates/{id}` | Inspect an in-memory template registry entry | | POST | `/v1/templates/gc` | Trigger template GC | +| GET | `/v1/runtime-templates` | List published runtime artifact sets | +| GET | `/v1/runtime-templates/{name}` | Inspect a published runtime artifact set | +| POST | `/v1/runtime-templates/import` | Publish artifacts from the configured import root | | GET | `/v1/policies` | List loaded policies | | GET | `/v1/hooks` | List kernel hooks | | GET | `/v1/metrics` | Prometheus metrics | | POST | `/v1/admin/reload` | Hot-reload policies | +The `/v1/runtime-templates` routes manage a durable artifact catalog that is +separate from the existing in-memory `/v1/templates` registry. Importing an +entry does not make sandbox creation select it. See +[Runtime template catalog](docs/design/runtime-template-catalog.md) for the +accepted artifacts, configuration limits, and publication rules. + ### Managed lifecycle and recovery Create and destroy record their operation before changing storage or backend diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index ff96fa68e7..d7dcd232a8 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -126,14 +126,22 @@ images_dir = "/var/lib/blaze/images" | GET | `/v1/pools/{backend}/{class}` | 获取 pool 状态 | | POST | `/v1/pools/{backend}/{class}/drain` | 排空 pool | | PUT | `/v1/pools/{backend}/{class}/sizing` | 调整 pool 大小 | -| GET | `/v1/templates` | 列出模板 | -| GET | `/v1/templates/{id}` | 查看模板详情 | +| GET | `/v1/templates` | 列出内存模板 registry 条目 | +| GET | `/v1/templates/{id}` | 查看内存模板 registry 条目 | | POST | `/v1/templates/gc` | 触发模板 GC | +| GET | `/v1/runtime-templates` | 列出已发布的 runtime artifact | +| GET | `/v1/runtime-templates/{name}` | 查看已发布的 runtime artifact | +| POST | `/v1/runtime-templates/import` | 从配置的导入根目录发布 artifact | | GET | `/v1/policies` | 列出已加载策略 | | GET | `/v1/hooks` | 列出内核 hook | | GET | `/v1/metrics` | Prometheus 指标 | | POST | `/v1/admin/reload` | 热加载策略 | +`/v1/runtime-templates` 路由管理持久 artifact 目录，与已有的内存 +`/v1/templates` registry 相互独立。导入条目不会让 sandbox create 自动 +选择它。接受的 artifact、配置上限和发布规则参见 +[Runtime 模板目录](docs/design/runtime-template-catalog_zh.md)。 + ### 生命周期管理与恢复 创建和销毁会在修改存储或后端资源之前记录当前操作。创建成功后状态为 diff --git a/src/blaze/docs/design/runtime-template-catalog.md b/src/blaze/docs/design/runtime-template-catalog.md new file mode 100644 index 0000000000..6f13ddb095 --- /dev/null +++ b/src/blaze/docs/design/runtime-template-catalog.md @@ -0,0 +1,88 @@ +# Runtime Template Catalog + +The daemon can publish a reusable runtime artifact set into the directory +configured by `runtime_templates.dir`. Imports are disabled unless an operator +also configures `runtime_templates.import_root`. + +The catalog is separate from the existing in-memory `/v1/templates` registry. +It provides durable publication and lookup, but sandbox creation does not +select catalog entries. + +## Import request + +```http +POST /v1/runtime-templates/import +Content-Type: application/json + +{ + "name": "runtime-base", + "source": "runtime-base", + "description": "base runtime" +} +``` + +`source` is a relative path below the configured import root. Absolute paths, +parent traversal, and symbolic links in the path are rejected. Every source +directory and file must be owned by the daemon user and must not be writable +by group or other users. + +The source must contain top-level regular files named `vmstate.snap`, +`mem.bin`, and `rootfs.ext4`. An optional `template.json` must contain a JSON +object. Nested directories, links, and special files are rejected. The daemon +sets `name` from the request, applies a non-empty request description, and +fills numeric `rootfs_size` and `memory_size` defaults when they are absent. +It returns `409 Conflict` when the destination exists or another import of the +same name is active. + +## Limits and owned paths + +The following settings bound work before data is published: + +| Setting | Meaning | +|---------|---------| +| `max_files` | Maximum files in one published entry, including `template.json` | +| `max_bytes` | Maximum artifact and generated metadata bytes in one entry | +| `max_metadata_bytes` | Maximum input and generated metadata size | +| `max_total_bytes` | Maximum committed bytes plus concurrent reservations | + +`runtime_templates.dir` and `runtime_templates.import_root` must be absolute, +must not contain parent components, and must not overlap each other. They also +must not overlap the storage image, storage instance, or configured template +directories. + +The catalog, staging directories, and published directories use mode `0700`. +Published files use mode `0600`. + +## Publication and recovery + +The importer opens source entries without following links, reserves catalog +capacity, and copies them into a private, uniquely named staging directory. +It checks the source identity and size again after copying. The complete +directory is synchronized and renamed into place without replacing an +existing entry, so readers see either no entry or the complete entry. + +A failed import removes its staging directory. If cleanup cannot be completed, +or publication has occurred but catalog durability cannot be confirmed, the +daemon rejects later imports until the catalog is repaired and the daemon +restarts. Startup removes owned staging directories left by an interrupted +run and validates the type, ownership, permissions, contents, and capacity of +published entries. + +During graceful shutdown, the daemon rejects new imports, requests +cancellation of active imports, waits for their file handles and staging data +to be released, and then continues normal runtime cleanup. + +## Lookup and current limits + +Published metadata is available through: + +- `GET /v1/runtime-templates` +- `GET /v1/runtime-templates/{name}` + +Catalog listing is sorted by template name and reports corrupt published +metadata instead of silently hiding it. These routes manage stored artifacts +only. Validation is structural; it does not prove that a snapshot is bootable +or compatible with a particular backend. This capability does not make +sandbox creation select an imported template, reference-count imported +entries, or remove their directories through the existing `/v1/templates/gc` +registry route. diff --git a/src/blaze/docs/design/runtime-template-catalog_zh.md b/src/blaze/docs/design/runtime-template-catalog_zh.md new file mode 100644 index 0000000000..1bd351b7b1 --- /dev/null +++ b/src/blaze/docs/design/runtime-template-catalog_zh.md @@ -0,0 +1,76 @@ +# Runtime 模板目录 + +daemon 可以将一组可复用的 runtime artifact 发布到 +`runtime_templates.dir` 配置的目录。只有同时配置 +`runtime_templates.import_root` 后，导入功能才会启用。 + +该目录与已有的内存 `/v1/templates` registry 相互独立。它提供持久化发布和 +查询，但 sandbox create 不会选择其中的条目。 + +## 导入请求 + +```http +POST /v1/runtime-templates/import +Content-Type: application/json + +{ + "name": "runtime-base", + "source": "runtime-base", + "description": "base runtime" +} +``` + +`source` 是配置的导入根目录下的相对路径。绝对路径、父目录跳转和路径中的 +符号链接都会被拒绝。每一级源目录和源文件都必须属于 daemon 用户，并且不能 +允许 group 或其他用户写入。 + +源目录必须包含顶层普通文件 `vmstate.snap`、`mem.bin` 和 `rootfs.ext4`。 +可选的 `template.json` 必须是 JSON object。嵌套目录、链接和特殊文件都会 +被拒绝。daemon 会使用请求中的 `name`，采用请求中非空的 `description`， +并在缺少数值类型的 `rootfs_size` 或 `memory_size` 时填入默认值。目标名称 +已存在或同名导入正在执行时返回 `409 Conflict`。 + +## 上限和目录边界 + +以下配置会在发布前限制一次导入所做的工作： + +| 配置 | 含义 | +|------|------| +| `max_files` | 单个发布条目的文件数上限，包括 `template.json` | +| `max_bytes` | 单个条目的 artifact 与生成后元数据的总字节数上限 | +| `max_metadata_bytes` | 输入与生成后元数据的大小上限 | +| `max_total_bytes` | 已发布字节与并发预留字节之和的上限 | + +`runtime_templates.dir` 与 `runtime_templates.import_root` 必须是绝对路径， +不能包含父目录组件，也不能互相重叠。它们还不能与存储镜像、存储实例或已有 +模板配置目录重叠。 + +catalog、staging 目录和已发布目录的权限为 `0700`，已发布文件的权限为 +`0600`。 + +## 发布与恢复 + +导入器打开源条目时不会跟随链接；它会先预留 catalog 容量，再把文件复制到 +私有且名称唯一的 staging 目录。复制完成后会再次检查源文件的身份和大小。 +完整目录同步后以不覆盖已有条目的方式改名发布，因此读取方只会看到“没有条目” +或完整条目。 + +导入失败会移除对应的 staging 目录。如果清理无法完成，或者条目已经发布但 +catalog 持久性无法确认，daemon 会拒绝后续导入，直到修复 catalog 并重启。 +启动时，daemon 会移除上次中断后遗留且归自己所有的 staging 目录，并校验 +已发布条目的类型、所有者、权限、内容和容量。 + +正常关闭时，daemon 会拒绝新导入，请求取消正在执行的导入，等待相关文件句柄 +和 staging 数据释放，然后继续既有的 runtime 清理。 + +## 查询与当前限制 + +已发布的元数据可通过以下接口查询： + +- `GET /v1/runtime-templates` +- `GET /v1/runtime-templates/{name}` + +列表按模板名称排序；已发布元数据损坏时会返回错误，而不是静默隐藏条目。这些 +接口只管理已经保存的 artifact，校验范围仅限结构，不证明快照能够启动或与某个 +backend 兼容。它不会让 sandbox create 自动选择导入模板，不会为导入条目维护 +引用计数，也不会通过既有 `/v1/templates/gc` registry 路由删除其目录。 From 33f1a64718f5dcc9c241cb43d4d41206e891e8e9 Mon Sep 17 00:00:00 2001 From: Weisson Date: Sat, 1 Aug 2026 07:40:50 +0800 Subject: [PATCH 39/40] fix(blaze): preserve network slot on restore This enables checkpoint rollback and hibernation resume to recreate the host-side device names embedded in a VM snapshot before loading it. Checkpoint and hibernation manifests now record guest transport and the exact network slot. The Firecracker adapter reserves that slot and fails without selecting a different one when the name is unavailable. Either restore path therefore keeps its recoverable failure state instead of loading a snapshot against mismatched devices. This restore invariant spans the shared adapter and both production consumers. It does not change ordinary cold or warm sandbox network allocation. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/crates/blaze-core/src/backend.rs | 2 + src/blaze/crates/blaze-core/src/checkpoint.rs | 14 +++ .../crates/blazed/src/checkpoint_store.rs | 4 + .../crates/blazed/src/sandbox/checkpoint.rs | 4 + .../crates/blazed/src/sandbox/hibernate.rs | 46 ++++++++ .../crates/blazed/src/sandbox/restore.rs | 7 +- src/blaze/crates/blazed/src/spawner.rs | 6 ++ .../crates/blazed/src/spawner/firecracker.rs | 80 ++++++++++++-- src/blaze/crates/blazed/src/spawner/netns.rs | 101 +++++++++++++++++- 9 files changed, 248 insertions(+), 16 deletions(-) diff --git a/src/blaze/crates/blaze-core/src/backend.rs b/src/blaze/crates/blaze-core/src/backend.rs index 30ebeeda5b..475ed61bac 100644 --- a/src/blaze/crates/blaze-core/src/backend.rs +++ b/src/blaze/crates/blaze-core/src/backend.rs @@ -135,6 +135,8 @@ pub struct RestoreRequest { pub snapshot_kind: SnapshotKind, /// Whether the captured runtime exposed the stable run-directory guest transport. pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, } /// Snapshot flavor requested from a backend. diff --git a/src/blaze/crates/blaze-core/src/checkpoint.rs b/src/blaze/crates/blaze-core/src/checkpoint.rs index 22eb299b73..4c11a28e30 100644 --- a/src/blaze/crates/blaze-core/src/checkpoint.rs +++ b/src/blaze/crates/blaze-core/src/checkpoint.rs @@ -55,6 +55,10 @@ pub struct CheckpointMetadata { pub created_at: DateTime, /// Backend snapshot semantics. pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable guest transport. + pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, /// Integrity records for all required artifacts. pub artifacts: Vec, } @@ -91,6 +95,10 @@ pub struct CommitCheckpoint { pub backend_version: Option, /// Backend snapshot semantics. pub snapshot_kind: SnapshotKind, + /// Whether the captured runtime exposed the stable guest transport. + pub expose_guest_socket: bool, + /// Stable host-network slot whose device names are embedded in the snapshot. + pub network_slot: Option, } /// Pure validation failure for a checkpoint identifier or manifest. @@ -371,6 +379,8 @@ mod tests { backend_version: Some("mock-v1".to_string()), created_at: Utc::now(), snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, artifacts: vec![ artifact("vmstate.snap", 'a'), artifact("memory.snap", 'b'), @@ -447,6 +457,8 @@ mod tests { backend: BackendKind::Mock, backend_version: None, snapshot_kind: SnapshotKind::Full, + expose_guest_socket: false, + network_slot: None, }; assert!(validate_commit_checkpoint(&id, &input).is_err()); } @@ -473,6 +485,8 @@ mod tests { backend: metadata.backend, backend_version: metadata.backend_version, snapshot_kind: metadata.snapshot_kind, + expose_guest_socket: metadata.expose_guest_socket, + network_slot: metadata.network_slot, }; let error = validate_commit_checkpoint(&metadata.id, &input) .expect_err("missing version must fail"); diff --git a/src/blaze/crates/blazed/src/checkpoint_store.rs b/src/blaze/crates/blazed/src/checkpoint_store.rs index 324c6411bb..32b4005c66 100644 --- a/src/blaze/crates/blazed/src/checkpoint_store.rs +++ b/src/blaze/crates/blazed/src/checkpoint_store.rs @@ -153,6 +153,8 @@ impl CheckpointStore { backend_version: input.backend_version, created_at: Utc::now(), snapshot_kind: input.snapshot_kind, + expose_guest_socket: input.expose_guest_socket, + network_slot: input.network_slot, artifacts, }; validate_checkpoint_manifest(&metadata, stage.sandbox_id, &stage.id)?; @@ -971,6 +973,8 @@ mod tests { backend: BackendKind::Mock, backend_version: Some("mock-v1".to_string()), snapshot_kind: SnapshotKind::Full, + expose_guest_socket: true, + network_slot: None, } } diff --git a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs index 27fb5f7b80..1e1348351c 100644 --- a/src/blaze/crates/blazed/src/sandbox/checkpoint.rs +++ b/src/blaze/crates/blazed/src/sandbox/checkpoint.rs @@ -57,6 +57,8 @@ impl SandboxManager { } self.require_live_backend(id, &backend).await?; let storage = self.storage.reconstruct(&id.to_string()).await?; + let expose_guest_socket = !backend.guest_socket_path().as_os_str().is_empty(); + let network_slot = backend.network_slot(); let stage = self.checkpoints.begin(id).map_err(checkpoint_store_error)?; let checkpoint_id = stage.id().to_string(); @@ -190,6 +192,8 @@ impl SandboxManager { backend: instance.backend, backend_version, snapshot_kind: SnapshotKind::Full, + expose_guest_socket, + network_slot, }, ) .map_err(checkpoint_store_error); diff --git a/src/blaze/crates/blazed/src/sandbox/hibernate.rs b/src/blaze/crates/blazed/src/sandbox/hibernate.rs index 3a01279041..4da0878b20 100644 --- a/src/blaze/crates/blazed/src/sandbox/hibernate.rs +++ b/src/blaze/crates/blazed/src/sandbox/hibernate.rs @@ -50,6 +50,7 @@ struct HibernateManifest { backend_version: Option, snapshot_kind: SnapshotKind, expose_guest_socket: bool, + network_slot: Option, artifacts: Vec, } @@ -111,6 +112,7 @@ impl SandboxManager { } let storage = self.storage.reconstruct(&id.to_string()).await?; let expose_guest_socket = !backend.guest_socket_path().as_os_str().is_empty(); + let network_slot = backend.network_slot(); let hibernate_dir = self.hibernate_dir(id); if let Err(error) = prepare_hibernate_directory(&hibernate_dir).await { let recovery = self.mark_instance_recovery(instance).err(); @@ -229,6 +231,7 @@ impl SandboxManager { &instance, capability.version, expose_guest_socket, + network_slot, ) .await { @@ -477,6 +480,7 @@ impl SandboxManager { expected_version: manifest.backend_version.clone(), snapshot_kind: manifest.snapshot_kind, expose_guest_socket: manifest.expose_guest_socket, + network_slot: manifest.network_slot, }) .await } @@ -787,6 +791,7 @@ async fn build_hibernate_manifest( instance: &SandboxInstance, backend_version: Option, expose_guest_socket: bool, + network_slot: Option, ) -> Result { let mut artifacts = Vec::with_capacity(REQUIRED_ARTIFACTS.len()); for name in REQUIRED_ARTIFACTS { @@ -801,6 +806,7 @@ async fn build_hibernate_manifest( backend_version, snapshot_kind: SnapshotKind::Full, expose_guest_socket, + network_slot, artifacts, }) } @@ -1014,3 +1020,43 @@ fn with_recovery_error( None => error, } } + +#[cfg(test)] +mod tests { + use blaze_core::backend::BackendKind; + use blaze_core::lifecycle::StartPath; + use blaze_core::policy::WorkloadClass; + + use super::*; + + #[tokio::test] + async fn manifest_preserves_the_network_slot_for_resume() { + let temp = tempfile::tempdir().expect("temp"); + tokio::fs::write(temp.path().join(VMSTATE_ARTIFACT), b"vmstate") + .await + .expect("VM state"); + tokio::fs::write(temp.path().join(MEMORY_ARTIFACT), b"memory") + .await + .expect("memory"); + let instance = SandboxInstance::new( + BackendKind::Firecracker, + WorkloadClass::AgentTool, + "sha256:image".to_string(), + StartPath::Cold, + "default".to_string(), + ); + + let manifest = build_hibernate_manifest( + temp.path(), + &instance, + Some("Firecracker v1.16.0".to_string()), + true, + Some(7), + ) + .await + .expect("manifest"); + + assert!(manifest.expose_guest_socket); + assert_eq!(manifest.network_slot, Some(7)); + } +} diff --git a/src/blaze/crates/blazed/src/sandbox/restore.rs b/src/blaze/crates/blazed/src/sandbox/restore.rs index 63c4044e37..ceefb7669b 100644 --- a/src/blaze/crates/blazed/src/sandbox/restore.rs +++ b/src/blaze/crates/blazed/src/sandbox/restore.rs @@ -120,8 +120,6 @@ impl SandboxManager { ))); } let storage = self.storage.reconstruct(&id.to_string()).await?; - let expose_guest_socket = !current_backend.guest_socket_path().as_os_str().is_empty(); - instance.begin_restore_operation(request.checkpoint_id.clone())?; crate::failpoint::state("restore-begin-state") .and_then(|_| self.persist_and_retain(instance.clone()))?; @@ -263,7 +261,8 @@ impl SandboxManager { checkpoint_backend: target.metadata.backend, expected_version: target.metadata.backend_version.clone(), snapshot_kind: target.metadata.snapshot_kind, - expose_guest_socket, + expose_guest_socket: target.metadata.expose_guest_socket, + network_slot: target.metadata.network_slot, }) .await } @@ -322,7 +321,7 @@ impl SandboxManager { )); } if let Err(error) = self - .verify_restored_backend(id, &restored, expose_guest_socket) + .verify_restored_backend(id, &restored, target.metadata.expose_guest_socket) .await { return Err(self.fail_after_restore_stop( diff --git a/src/blaze/crates/blazed/src/spawner.rs b/src/blaze/crates/blazed/src/spawner.rs index 2b0492e4db..0cd0a06879 100644 --- a/src/blaze/crates/blazed/src/spawner.rs +++ b/src/blaze/crates/blazed/src/spawner.rs @@ -70,6 +70,10 @@ pub trait BackendInstance: Send + Sync { fn guest_socket_path(&self) -> &Path { Path::new("") } + /// Stable host-network slot required to reproduce snapshot device names. + fn network_slot(&self) -> Option { + None + } /// Report an observed backend exit without waiting. /// /// `None` means the owned process or task was running when checked. @@ -417,6 +421,7 @@ impl BackendSpawner for MockSpawner { expected_version, snapshot_kind, expose_guest_socket, + network_slot: _, .. } = request; if checkpoint_backend != BackendKind::Mock @@ -1266,6 +1271,7 @@ mod tests { expected_version: None, snapshot_kind: SnapshotKind::Full, expose_guest_socket: true, + network_slot: None, }; assert!( diff --git a/src/blaze/crates/blazed/src/spawner/firecracker.rs b/src/blaze/crates/blazed/src/spawner/firecracker.rs index 4eb4b67e7a..aaf8138486 100644 --- a/src/blaze/crates/blazed/src/spawner/firecracker.rs +++ b/src/blaze/crates/blazed/src/spawner/firecracker.rs @@ -152,13 +152,23 @@ impl FirecrackerSpawner { } .into()); } - match self - .network - .create(request.instance_id, |slot| { - write_network_metadata(&network_file, slot) - }) - .await - { + let created = match restore.as_ref().and_then(|restore| restore.network_slot) { + Some(slot) => { + self.network + .create_at(request.instance_id, slot, |slot| { + write_network_metadata(&network_file, slot) + }) + .await + } + None => { + self.network + .create(request.instance_id, |slot| { + write_network_metadata(&network_file, slot) + }) + .await + } + }; + match created { Ok(network) => Some(network), Err(error) => { let (source, residual) = error.into_parts(); @@ -457,17 +467,24 @@ impl BackendSpawner for FirecrackerSpawner { expected_version, snapshot_kind, expose_guest_socket, + network_slot, } = request; + let backend = blaze_core::policy::BackendConfigs { + firecracker: Some(blaze_core::policy::FirecrackerConfig { + enable_vsock: expose_guest_socket, + enable_network: network_slot.is_some(), + ..blaze_core::policy::FirecrackerConfig::default() + }), + }; self.start( SpawnRequest { instance_id, run_dir, binary_path, storage, - // Snapshot restore does not reconstruct the original policy. - // Keep host-side logging disabled unless a future restore - // contract models that setting explicitly. - backend: blaze_core::policy::BackendConfigs::default(), + // Snapshot restore only reconstructs host-side resources that + // are required to make the captured runtime reachable. + backend, vm: None, }, Some(FirecrackerRestore { @@ -477,6 +494,7 @@ impl BackendSpawner for FirecrackerSpawner { expected_version, snapshot_kind, expose_guest_socket, + network_slot, }), ) .await @@ -511,6 +529,7 @@ struct FirecrackerInstance { files: FirecrackerRuntimeFiles, guest_socket: PathBuf, network: Mutex>, + network_slot: Option, network_manager: Arc, cleanup_complete: AtomicBool, killed: AtomicBool, @@ -551,6 +570,7 @@ impl FirecrackerInstance { enable_vsock: bool, ) -> Self { let guest_socket = configured_guest_socket(enable_vsock, files.guest_socket.clone()); + let network_slot = network.as_ref().map(NetworkSlot::slot); Self { instance_id, child: Mutex::new(child), @@ -559,6 +579,7 @@ impl FirecrackerInstance { files, guest_socket, network: Mutex::new(network), + network_slot, network_manager, cleanup_complete: AtomicBool::new(false), killed: AtomicBool::new(false), @@ -592,6 +613,10 @@ impl BackendInstance for FirecrackerInstance { &self.guest_socket } + fn network_slot(&self) -> Option { + self.network_slot + } + async fn try_wait(&self) -> Result> { let result = { let mut guard = self.child.lock().await; @@ -708,6 +733,7 @@ struct FirecrackerRestore { expected_version: Option, snapshot_kind: SnapshotKind, expose_guest_socket: bool, + network_slot: Option, } impl FirecrackerCapture { @@ -1384,6 +1410,7 @@ mod tests { expected_version: Some("Firecracker v1.16.0".to_string()), snapshot_kind: SnapshotKind::Full, expose_guest_socket: false, + network_slot: None, }; validate_restore_compatibility(&restore, "Firecracker v1.16.0").expect("matching version"); @@ -1410,6 +1437,33 @@ mod tests { ); } + #[test] + fn instance_preserves_its_network_slot_for_restore() { + let temp = tempfile::tempdir().expect("temp"); + let api_socket = temp.path().join("api.sock"); + let instance = FirecrackerInstance::new( + Uuid::new_v4(), + None, + FirecrackerCapture::new( + api_socket.clone(), + Duration::from_secs(1), + "Firecracker v1.16.0".to_string(), + ), + runtime_files( + api_socket, + temp.path().join("vsock.uds"), + temp.path().join("firecracker.pid"), + stopped_marker(temp.path()), + temp.path().join("network.json"), + ), + Some(test_network_slot(7)), + Arc::new(NetworkManager::default()), + false, + ); + + assert_eq!(instance.network_slot(), Some(7)); + } + #[cfg(target_os = "linux")] async fn spawn_api( socket: &Path, @@ -1626,6 +1680,7 @@ mod tests { expected_version: Some("Firecracker v1.16.0".to_string()), snapshot_kind: SnapshotKind::Full, expose_guest_socket: false, + network_slot: None, }) .await { @@ -1683,6 +1738,7 @@ mod tests { expected_version: Some("Firecracker v1.16.0".to_string()), snapshot_kind: SnapshotKind::Full, expose_guest_socket: false, + network_slot: None, }) .await .expect("load snapshot"); @@ -1748,6 +1804,7 @@ mod tests { expected_version: Some("Firecracker v1.16.0".to_string()), snapshot_kind: SnapshotKind::Full, expose_guest_socket: true, + network_slot: None, }) .await .expect("load snapshot"); @@ -1815,6 +1872,7 @@ mod tests { expected_version: Some("Firecracker v1.16.0".to_string()), snapshot_kind: SnapshotKind::Full, expose_guest_socket: true, + network_slot: None, }; let load_error = instance diff --git a/src/blaze/crates/blazed/src/spawner/netns.rs b/src/blaze/crates/blazed/src/spawner/netns.rs index 24441a51fd..02a4d9a625 100644 --- a/src/blaze/crates/blazed/src/spawner/netns.rs +++ b/src/blaze/crates/blazed/src/spawner/netns.rs @@ -243,6 +243,32 @@ impl NetworkManager { pub(super) async fn create( &self, owner: Uuid, + record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + self.create_with_slot(owner, None, record).await + } + + /// Recreate the exact slot whose device names are embedded in a snapshot. + pub(super) async fn create_at( + &self, + owner: Uuid, + slot: usize, + record: F, + ) -> std::result::Result + where + F: FnMut(&NetworkSlot) -> Result<()>, + { + NetworkSlot::from_record(slot, owner)?; + self.create_with_slot(owner, Some(slot), record).await + } + + async fn create_with_slot( + &self, + owner: Uuid, + requested_slot: Option, mut record: F, ) -> std::result::Result where @@ -251,7 +277,10 @@ impl NetworkManager { let _host_guard = self.acquire_host_guard().await?; let mut blocked = self.existing_slots().await?; let (slot, network) = loop { - let slot = self.allocate(&blocked)?; + let slot = match requested_slot { + Some(slot) => self.reserve(slot, &blocked)?, + None => self.allocate(&blocked)?, + }; let network = network_slot(slot, owner); let add_namespace = vec!["netns".into(), "add".into(), network.netns.clone()]; match self.run_ip(&add_namespace).await { @@ -265,6 +294,13 @@ impl NetworkManager { self.release(slot); let refreshed = self.existing_slots().await?; if refreshed.contains(&slot) { + if requested_slot.is_some() { + return Err(NetworkCreateError::clean(BlazeError::BackendError { + msg: format!( + "required network slot {slot} is unavailable during restore" + ), + })); + } blocked.extend(refreshed); continue; } @@ -422,6 +458,29 @@ impl NetworkManager { }) } + fn reserve(&self, slot: usize, blocked: &HashSet) -> Result { + if slot >= NET_MAX_SLOT { + return Err(BlazeError::BackendError { + msg: format!("network slot {slot} is outside 0..{NET_MAX_SLOT}"), + }); + } + if blocked.contains(&slot) { + return Err(BlazeError::BackendError { + msg: format!("required network slot {slot} is unavailable during restore"), + }); + } + let mut state = self.state.lock().map_err(|_| BlazeError::BackendError { + msg: "network slot allocator lock poisoned".to_string(), + })?; + if !state.used.insert(slot) { + return Err(BlazeError::BackendError { + msg: format!("required network slot {slot} is already reserved"), + }); + } + state.next = (slot + 1) % NET_MAX_SLOT; + Ok(slot) + } + async fn existing_slots(&self) -> Result> { let args = vec!["netns".to_string(), "list".to_string()]; let output = self.run_ip_output(&args).await?; @@ -815,6 +874,46 @@ mod tests { ); } + #[tokio::test] + async fn restore_recreates_the_requested_network_slot() { + let runner = Arc::new(FakeIpRunner::default()); + let manager = NetworkManager::with_runner(runner.clone()); + manager.state.lock().expect("state lock").next = 11; + + let slot = manager + .create_at(Uuid::from_u128(1), 7, |_| Ok(())) + .await + .expect("restore slot"); + + assert_eq!(slot.slot(), 7); + assert!( + runner + .calls() + .iter() + .any(|args| args == &["netns", "add", test_network_slot(7).netns()]) + ); + } + + #[tokio::test] + async fn restore_rejects_an_occupied_network_slot() { + let namespace = format!("{}\n", test_network_slot(7).netns()); + let runner = Arc::new(FakeIpRunner::with_responses([success( + namespace.as_bytes(), + )])); + let manager = NetworkManager::with_runner(runner.clone()); + + let error = manager + .create_at(Uuid::from_u128(1), 7, |_| Ok(())) + .await + .expect_err("occupied restore slot"); + + assert!(error.to_string().contains("slot 7 is unavailable")); + assert_eq!( + runner.calls(), + vec![vec!["netns".to_string(), "list".to_string()]] + ); + } + #[tokio::test] async fn create_retries_when_namespace_appears_during_allocation() { let runner = Arc::new(FakeIpRunner::with_responses([ From 828f73d286ef941b9f4f5c26ebb0d515410001fb Mon Sep 17 00:00:00 2001 From: Weisson Date: Tue, 28 Jul 2026 16:49:55 +0800 Subject: [PATCH 40/40] chore(blaze): bump version to 0.4.0 This enables maintainers and operators to identify this daemon release as 0.4.0 through blazed --version, GET /v1/health, Cargo metadata, RPM metadata, both component manifests, and the English and Chinese release notes. All version-bearing files change together so a checked-out tree cannot report different Blaze versions through different entry points. The release notes describe the live capability boundaries inherited from the release base; this commit changes no runtime behavior. The version remains duplicated across several repository formats and therefore still requires an explicit consistency check before publication. Signed-off-by: Jing Zhang Signed-off-by: Weisson --- src/blaze/.anolisa/component.toml | 2 +- src/blaze/CHANGELOG.md | 20 +++++++++++++++++++- src/blaze/CHANGELOG_zh.md | 19 +++++++++++++++++++ src/blaze/Cargo.lock | 4 ++-- src/blaze/Cargo.toml | 2 +- src/blaze/README.md | 4 ++-- src/blaze/README_zh.md | 4 ++-- src/blaze/dist/blaze.spec | 2 +- src/blaze/manifests/blaze.toml | 2 +- 9 files changed, 48 insertions(+), 11 deletions(-) diff --git a/src/blaze/.anolisa/component.toml b/src/blaze/.anolisa/component.toml index 727cddd765..4b88db4f22 100644 --- a/src/blaze/.anolisa/component.toml +++ b/src/blaze/.anolisa/component.toml @@ -1,6 +1,6 @@ [component] name = "blaze" -version = "0.3.0" +version = "0.4.0" type = "daemon" language = "rust" description = "Per-host sandbox orchestrator daemon" diff --git a/src/blaze/CHANGELOG.md b/src/blaze/CHANGELOG.md index 4c70d5a5d7..dd87e62b4f 100644 --- a/src/blaze/CHANGELOG.md +++ b/src/blaze/CHANGELOG.md @@ -9,6 +9,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.4.0] - 2026-08-01 + +### Added + +- Sandbox create, list, inspect, and destroy APIs backed by persistent lifecycle operations and an explicit recovery state. +- Bounded guest command execution and file read/write APIs with explicit payload and request limits. +- A configurable runtime warm pool with asynchronous refill, capacity reporting, cleanup, and optional ready virtual machines. +- Optional per-sandbox networking for Firecracker virtual machines, with isolated allocation and owned cleanup. +- Integrity-checked checkpoint artifacts with lineage, rollback, and pruning. +- Sandbox hibernate and resume operations with durable artifacts and restart recovery. +- A transactional runtime artifact catalog with bounded import validation and daemon query APIs. +- Optional periodic synchronization of storage owned by running sandboxes. + +### Changed + +- The file provider now creates self-contained runtime snapshots, using additional capacity so each restore remains independent. +- Runtime processes and host resources remain behind backend ownership interfaces, allowing interrupted cleanup to be retried. +- Network-enabled VM snapshots restore the captured network slot so guest device names remain stable. + ## [0.3.0] - 2026-07-22 ### Added @@ -73,4 +92,3 @@ Initial scaffold of ANOLISA Anvil per-host sandbox daemon. - Warm pool: pre-created sandboxes ready for instant allocation, configurable min/target/max. - Template sharing: multiple sandboxes share one base memory image, reducing per-instance cost. - Prometheus metrics endpoint for monitoring. - diff --git a/src/blaze/CHANGELOG_zh.md b/src/blaze/CHANGELOG_zh.md index 2be13a6ae9..a6a77d9fbc 100644 --- a/src/blaze/CHANGELOG_zh.md +++ b/src/blaze/CHANGELOG_zh.md @@ -9,6 +9,25 @@ ## [未发布] +## [0.4.0] - 2026-08-01 + +### 新增 + +- Sandbox 创建、列出、查看和销毁 API，并通过持久化生命周期操作与显式恢复状态处理失败。 +- 有界 Guest 命令执行和文件读写 API，并具有明确的载荷与请求限制。 +- 可配置的运行时 warm pool，支持异步补充、容量报告、清理以及可选的就绪虚拟机。 +- 可选的 Firecracker sandbox 独立网络，并对已分配资源承担清理责任。 +- 带完整性校验和谱系关系的 checkpoint 制品，支持 rollback 与 prune。 +- 使用持久化制品且可在守护进程重启后恢复的 sandbox hibernate 与 resume。 +- 具有有界导入校验和 daemon 查询 API 的事务式运行制品目录。 +- 可选的运行中 sandbox 存储定期同步。 + +### 变更 + +- File provider 现在创建自包含运行时快照，以额外容量换取每次 restore 的独立性。 +- 运行时进程和 host 资源由 backend 所有权接口持有，中断的清理操作可以重试。 +- 启用网络的 VM snapshot 会恢复 capture 时的网络 slot，保持 guest 设备名稳定。 + ## [0.3.0] - 2026-07-22 ### 新增 diff --git a/src/blaze/Cargo.lock b/src/blaze/Cargo.lock index 152ccdc5c6..c13c21555d 100644 --- a/src/blaze/Cargo.lock +++ b/src/blaze/Cargo.lock @@ -113,7 +113,7 @@ checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" [[package]] name = "blaze-core" -version = "0.3.0" +version = "0.4.0" dependencies = [ "async-trait", "chrono", @@ -129,7 +129,7 @@ dependencies = [ [[package]] name = "blazed" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "async-trait", diff --git a/src/blaze/Cargo.toml b/src/blaze/Cargo.toml index 1a23d8ad02..11192e2e95 100644 --- a/src/blaze/Cargo.toml +++ b/src/blaze/Cargo.toml @@ -7,7 +7,7 @@ members = [ ] [workspace.package] -version = "0.3.0" +version = "0.4.0" edition = "2024" license = "Apache-2.0" rust-version = "1.88" diff --git a/src/blaze/README.md b/src/blaze/README.md index 8af9ab62e9..345c7ca74f 100644 --- a/src/blaze/README.md +++ b/src/blaze/README.md @@ -405,8 +405,8 @@ readiness: ```json { "status": "ok", - "version": "0.3.0", - "storage_pool": { "ready": 0, "capacity": 0, "pending": 0 } + "version": "0.4.0", + "storage_pool": { "ready": 0, "capacity": 0, "pending": 0, "quarantined": 0 } } ``` diff --git a/src/blaze/README_zh.md b/src/blaze/README_zh.md index d629c10984..316471bfb3 100644 --- a/src/blaze/README_zh.md +++ b/src/blaze/README_zh.md @@ -361,8 +361,8 @@ read 返回内容过大时，API 返回 HTTP 502 和 ```json { "status": "ok", - "version": "0.3.0", - "storage_pool": { "ready": 0, "capacity": 0, "pending": 0 } + "version": "0.4.0", + "storage_pool": { "ready": 0, "capacity": 0, "pending": 0, "quarantined": 0 } } ``` diff --git a/src/blaze/dist/blaze.spec b/src/blaze/dist/blaze.spec index 3fe3ad17c2..b0dc968903 100644 --- a/src/blaze/dist/blaze.spec +++ b/src/blaze/dist/blaze.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} Name: blaze -Version: 0.3.0 +Version: 0.4.0 Release: %{anolis_release}%{?dist} Summary: Per-host sandbox orchestrator daemon for AI Agent workloads diff --git a/src/blaze/manifests/blaze.toml b/src/blaze/manifests/blaze.toml index 77434cdee9..f87c4cd2f6 100644 --- a/src/blaze/manifests/blaze.toml +++ b/src/blaze/manifests/blaze.toml @@ -3,7 +3,7 @@ [component] name = "blaze" -version = "0.3.0" +version = "0.4.0" layer = "runtime" domain = "blaze" description = "Per-host sandbox orchestrator — multi-backend lifecycle management, kernel coordination"