From 072d487f625d4bad1db8efa177ab51e1a5da3f06 Mon Sep 17 00:00:00 2001 From: Aiden McClelland Date: Wed, 26 Aug 2026 12:57:21 -0600 Subject: [PATCH 1/2] fix(core): don't convert the transfer source drive to btrfs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `migrate` imported the source drive through the same `disk::main::import` every other flow uses, and `mount_fs` converts an ext4 data partition to btrfs in place whenever it mounts one. So a transfer off a 0.3.x drive ran e2fsck, `btrfs-convert`, `btrfs subvolume delete ext2_saved` and a full recursive defragment against the drive the user is keeping as their fallback — before it read a byte of their data — and a failure partway through could have taken the original with it. Conversion is now a decision the caller makes: `Ext4Conversion::Convert` for a drive the server takes ownership of (boot, attach, a new data drive, the setup target) and `Preserve` for the transfer source. The source is still fsck'd (that is what makes the mount safe) and still has its `package-data/tmp` removed so transient data isn't copied across. A fully read-only source mount would need that tmp handling reworked. --- .../crates/start-core/src/bins/start_init.rs | 3 +- .../crates/start-core/src/disk/main.rs | 33 ++++++++++++++++--- shared-libs/crates/start-core/src/setup.rs | 7 +++- 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/shared-libs/crates/start-core/src/bins/start_init.rs b/shared-libs/crates/start-core/src/bins/start_init.rs index b47bfb0752..a8b9eed19e 100644 --- a/shared-libs/crates/start-core/src/bins/start_init.rs +++ b/shared-libs/crates/start-core/src/bins/start_init.rs @@ -6,7 +6,7 @@ use crate::context::rpc::InitRpcContextPhases; use crate::context::{DiagnosticContext, InitContext, RpcContext, SetupContext}; use crate::disk::REPAIR_DISK_PATH; use crate::disk::fsck::RepairStrategy; -use crate::disk::main::DEFAULT_PASSWORD; +use crate::disk::main::{DEFAULT_PASSWORD, Ext4Conversion}; use crate::firmware::{check_for_firmware_update, update_firmware}; use crate::init::{InitPhases, STANDBY_MODE_PATH}; use crate::net::gateway::WildcardListener; @@ -154,6 +154,7 @@ async fn setup_or_init( } else { RepairStrategy::Preen }, + Ext4Conversion::Convert, if disk_guid.ends_with("_UNENC") { None } else { diff --git a/shared-libs/crates/start-core/src/disk/main.rs b/shared-libs/crates/start-core/src/disk/main.rs index 9f82167c2d..23973a07ce 100644 --- a/shared-libs/crates/start-core/src/disk/main.rs +++ b/shared-libs/crates/start-core/src/disk/main.rs @@ -20,6 +20,15 @@ pub const PASSWORD_PATH: &'static str = "/run/startos/password"; pub const DEFAULT_PASSWORD: &'static str = "password"; pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8); +/// Whether mounting may convert an ext4 data partition to btrfs in place. +/// Conversion rewrites the partition, so a caller that only reads a drive it is +/// leaving behind must pass `Preserve`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Ext4Conversion { + Convert, + Preserve, +} + #[instrument(skip_all)] pub async fn create( disks: &I, @@ -216,6 +225,7 @@ pub async fn import>( guid: &str, datadir: P, repair: RepairStrategy, + conversion: Ext4Conversion, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { @@ -266,7 +276,7 @@ pub async fn import>( .arg(guid) .invoke(crate::ErrorKind::DiskManagement) .await?; - mount_all_fs(guid, datadir, repair, password, progress).await + mount_all_fs(guid, datadir, repair, conversion, password, progress).await } #[instrument(skip_all)] @@ -275,6 +285,7 @@ pub async fn mount_fs>( datadir: P, name: &str, repair: RepairStrategy, + conversion: Ext4Conversion, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { @@ -305,9 +316,8 @@ pub async fn mount_fs>( blockdev_path = Path::new("/dev/mapper").join(&full_name); } - // Convert ext4 → btrfs on the package-data partition if needed let fs_type = detect_filesystem(&blockdev_path).await?; - if fs_type == "ext2" { + if fs_type == "ext2" && conversion == Ext4Conversion::Convert { let mut convert_phase = progress.map(|p| p.add_phase(t!("disk.main.converting-to-btrfs").into(), Some(50))); if let Some(ref mut phase) = convert_phase { @@ -401,12 +411,25 @@ pub async fn mount_all_fs>( guid: &str, datadir: P, repair: RepairStrategy, + conversion: Ext4Conversion, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { let mut reboot = RequiresReboot(false); - reboot |= mount_fs(guid, &datadir, "main", repair, password, progress).await?; - reboot |= mount_fs(guid, &datadir, "package-data", repair, password, progress).await?; + reboot |= mount_fs( + guid, &datadir, "main", repair, conversion, password, progress, + ) + .await?; + reboot |= mount_fs( + guid, + &datadir, + "package-data", + repair, + conversion, + password, + progress, + ) + .await?; Ok(reboot) } diff --git a/shared-libs/crates/start-core/src/setup.rs b/shared-libs/crates/start-core/src/setup.rs index 8604f44eae..59af6bec7b 100644 --- a/shared-libs/crates/start-core/src/setup.rs +++ b/shared-libs/crates/start-core/src/setup.rs @@ -28,7 +28,7 @@ use crate::context::{CliContext, RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::REPAIR_DISK_PATH; use crate::disk::fsck::RepairStrategy; -use crate::disk::main::DEFAULT_PASSWORD; +use crate::disk::main::{DEFAULT_PASSWORD, Ext4Conversion}; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::filesystem::cifs::Cifs; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; @@ -300,6 +300,7 @@ pub async fn attach( } else { RepairStrategy::Preen }, + Ext4Conversion::Convert, if disk_guid.ends_with("_UNENC") { None } else { @@ -493,6 +494,7 @@ pub async fn setup_data_drive( &*guid, DATA_DIR, RepairStrategy::Preen, + Ext4Conversion::Convert, encryption_password, None, ) @@ -896,6 +898,7 @@ pub async fn execute_inner( } else { RepairStrategy::Preen }, + Ext4Conversion::Convert, if guid.ends_with("_UNENC") { None } else { @@ -1083,10 +1086,12 @@ async fn migrate( restore_phase.start(); restore_phase.set_units(Some(ProgressUnits::Bytes)); + // A transfer only reads the source drive, so it stays as the user's fallback. let _ = crate::disk::main::import( &old_guid, "/media/startos/migrate", RepairStrategy::Preen, + Ext4Conversion::Preserve, if guid.ends_with("_UNENC") { None } else { From c6d3d991bd618a34ff67f38e590505f04e3db4bd Mon Sep 17 00:00:00 2001 From: Helix <267227783+helix-nine@users.noreply.github.com> Date: Wed, 2 Sep 2026 19:05:59 +0000 Subject: [PATCH 2/2] fix(core): keep transfer sources read-only Mount transfer-source PVs, logical volumes, encrypted mappings, and filesystems read-only. Dirty ext4 and btrfs sources retry with journal or log replay disabled rather than repairing the source.\n\nExclude transient package data without deleting it, and always attempt full source teardown while preserving the transfer error as primary. The import mode now owns repair and mount policy, eliminating invalid combinations and the redundant filesystem probe. --- projects/start-os/CHANGELOG.md | 4 + projects/start-os/docs/src/initial-setup.md | 2 +- shared-libs/AGENTS.md | 2 +- shared-libs/CONTRIBUTING.md | 4 +- shared-libs/crates/start-core/AGENTS.md | 24 +- shared-libs/crates/start-core/ARCHITECTURE.md | 3 +- shared-libs/crates/start-core/CONTRIBUTING.md | 80 ---- shared-libs/crates/start-core/README.md | 7 +- .../crates/start-core/locales/i18n.yaml | 7 + .../crates/start-core/src/bins/start_init.rs | 7 +- .../crates/start-core/src/disk/fsck/ext4.rs | 16 +- .../crates/start-core/src/disk/main.rs | 409 ++++++++++++------ shared-libs/crates/start-core/src/setup.rs | 169 ++++---- shared-libs/crates/start-core/src/util/io.rs | 105 ++++- 14 files changed, 518 insertions(+), 321 deletions(-) delete mode 100644 shared-libs/crates/start-core/CONTRIBUTING.md diff --git a/projects/start-os/CHANGELOG.md b/projects/start-os/CHANGELOG.md index 33b6b5d352..3ac07e66e0 100644 --- a/projects/start-os/CHANGELOG.md +++ b/projects/start-os/CHANGELOG.md @@ -81,6 +81,10 @@ file tracks notable changes since the move to the monorepo. ### Fixed +- **Transfers preserve the source filesystem format.** StartOS mounts source + filesystems read-only while copying persistent data, repairing ext4 only when + needed to mount it. This leaves the source drive available as a fallback. + - **An app that remembers your server's certificate sees the same certificate across every route to that name.** Wallets and other apps that pin the first certificate they are shown — Sparrow and the Electrum clients most visibly — diff --git a/projects/start-os/docs/src/initial-setup.md b/projects/start-os/docs/src/initial-setup.md index de732a9bdc..6d81b46f49 100644 --- a/projects/start-os/docs/src/initial-setup.md +++ b/projects/start-os/docs/src/initial-setup.md @@ -18,7 +18,7 @@ After [installing StartOS](installing-startos.md), follow these steps to initial - **Restore from Backup**: Select this option _only_ if your existing StartOS data drive has been lost or corrupted. This is for disaster recovery only. The restored server keeps the source server's name. - - **Transfer**: Select this option if you are transferring your existing data from one drive to another. The transferred server keeps the source server's name. + - **Transfer**: Select this option if you are transferring your existing data from one drive to another. StartOS mounts the source filesystems read-only while copying. If an ext4 filesystem must be repaired before it can be mounted, StartOS repairs it and retries. The source drive remains a usable copy if the transfer fails. The transferred server keeps the source server's name. Once the new drive is running, do not boot the old one as a server again. 1. Set a strong master password. _Make it good. Write it down_. Resetting your password is non-trivial, but your data will be preserved. diff --git a/shared-libs/AGENTS.md b/shared-libs/AGENTS.md index e7e55fbf64..69a1177492 100644 --- a/shared-libs/AGENTS.md +++ b/shared-libs/AGENTS.md @@ -10,7 +10,7 @@ read its own `AGENTS.md` first. `CLAUDE.md` is a one-line `@AGENTS.md` import. S ## Layout - `crates/start-core/` — Rust backend lib (`start-core`, lib name `start_core`). - Has its own `AGENTS.md`, `ARCHITECTURE.md`, `CONTRIBUTING.md`, plus topic notes + Has its own `AGENTS.md` and `ARCHITECTURE.md`, plus topic notes (`core-rust-patterns.md`, `i18n-patterns.md`, `patchdb.md`, `rpc-toolkit.md`, `s9pk-structure.md`, `exver.md`, `VERSION_BUMP.md`). - `ts-modules/` — shared TypeScript modules; the `@start9labs/shared` diff --git a/shared-libs/CONTRIBUTING.md b/shared-libs/CONTRIBUTING.md index da0949ac93..4693b0438f 100644 --- a/shared-libs/CONTRIBUTING.md +++ b/shared-libs/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing to shared-libs `shared-libs/` groups two cross-product libraries. Contribute inside the relevant -sub-library; each has its own `CONTRIBUTING.md` with the full detail. +sub-library and follow its `AGENTS.md`. ## Documentation @@ -32,7 +32,7 @@ cd shared-libs/crates/start-core && ./run-tests.sh - Local `cargo check` is linux-only. CI builds an apple-darwin + linux-musl matrix; consider those targets for any change touching `libc`/platform APIs or dependencies (cfg-gate platform code rather than reimplementing it). -- See [`crates/start-core/CONTRIBUTING.md`](crates/start-core/CONTRIBUTING.md) +- See [`crates/start-core/AGENTS.md`](crates/start-core/AGENTS.md) and the topic notes (`core-rust-patterns.md`, `patchdb.md`, `rpc-toolkit.md`, `i18n-patterns.md`, `VERSION_BUMP.md`). diff --git a/shared-libs/crates/start-core/AGENTS.md b/shared-libs/crates/start-core/AGENTS.md index b291ff979a..70c6115638 100644 --- a/shared-libs/crates/start-core/AGENTS.md +++ b/shared-libs/crates/start-core/AGENTS.md @@ -5,8 +5,7 @@ All six product bins (`startbox`/`startd`, `start-container`, `start-cli`, `regi `tunnelbox`, `startwrt`) link against it; all but `startwrt` are thin wrappers in the product crates (`startwrt` is a full backend of its own that imports this crate aliased as `startos`). -`CLAUDE.md` is a one-line `@AGENTS.md` import. See [ARCHITECTURE.md](ARCHITECTURE.md) and -[CONTRIBUTING.md](CONTRIBUTING.md). +`CLAUDE.md` is a one-line `@AGENTS.md` import. See [ARCHITECTURE.md](ARCHITECTURE.md). Topical references: [rpc-toolkit.md](rpc-toolkit.md), [patchdb.md](patchdb.md), [i18n-patterns.md](i18n-patterns.md), [core-rust-patterns.md](core-rust-patterns.md), @@ -24,19 +23,34 @@ Topical references: [rpc-toolkit.md](rpc-toolkit.md), [patchdb.md](patchdb.md), ## Build & test (run from the repo root) +Install stable Rust and Docker. `rust-analyzer` is recommended. + - `cargo check -p start-core` — type-check the library. - `make start-core-test` — run the test suite (wraps `run-tests.sh`, which uses the `rust-zig-builder` container and the `test` feature; skips `export_` tests). Or run a single test directly: `cargo test -p start-core --features=test`. -- `make start-core-format` — format this crate (`make start-core-format-check` for the read-only CI check). Nightly is required for formatting. +- `make start-core-format` — format the shared Rust crates with the pinned nightly container (`make start-core-format-check` checks formatting). Set `FMT_NATIVE=1` only when that nightly is installed locally. - `cargo build -p start-os --bin startbox` (or the other product crate/bin) to build a binary. +Keep `README.md`, `ARCHITECTURE.md`, this file, and the topical references current when changing +project structure, conventions, build process, or product context. + +## Adding an RPC endpoint + +1. Define serializable request and response types. +2. Choose a handler type (`from_fn_async` for most cases). +3. Write the async handler. +4. Register it in the appropriate `ParentHandler` tree. +5. Add `#[derive(TS)]` and `#[ts(export)]` where TypeScript needs the types. + +See [rpc-toolkit.md](rpc-toolkit.md) for handler patterns. + ## Gotchas - **Process invocation: use `.invoke(ErrorKind::...)`, not `.status()`.** When running CLI commands via `tokio::process::Command`, the `Invoke` trait (from `crate::util::Invoke`) captures stdout/stderr and checks exit codes. `.status()` leaks stderr directly to system logs and creates noise in production. For check-then-act patterns (e.g. `iptables -C`), use `.invoke(...).await.is_ok()` / `.is_err()` instead of `.status().await.map_or(false, |s| s.success())`. - **File I/O: prefer `crate::util::io` over `tokio::fs`** when an equivalent helper exists. These helpers add error context and mount-aware behavior that `tokio::fs` doesn't. -- **i18n is mandatory for any user-facing string** — including CLI subcommand descriptions (`about.`), CLI arg help (`help.arg.`), error messages, notifications, and setup messages. All 5 locales (`en_US`, `de_DE`, `es_ES`, `fr_FR`, `pl_PL`) must be filled in `locales/i18n.yaml` (i.e. `shared-libs/crates/start-core/locales/i18n.yaml`), alphabetically ordered within their section. See `i18n-patterns.md`. Compile-time validation catches missing keys. -- **`#[ts(export)]` changes need a multi-step rebuild.** Editing a `#[ts(export)]` struct/enum in Rust does _not_ update web or container-runtime. From the repo root, run `make start-core-ts-bindings` (regenerates `shared-libs/crates/start-core/bindings/`, then rsyncs into `shared-libs/ts-modules/start-core/lib/osBindings/`) and then rebuild the affected TS lib(s): `cd shared-libs/ts-modules/start-core && make dist` (what web consumes) and/or `cd projects/start-sdk && make bundle` (the SDK bundle container-runtime consumes). See [ARCHITECTURE.md](ARCHITECTURE.md#cross-layer-verification). +- **i18n is mandatory for any user-facing string** — including CLI subcommand descriptions (`about.`), CLI arg help (`help.arg.`), error messages, notifications, and setup messages. All 5 locales (`en_US`, `de_DE`, `es_ES`, `fr_FR`, `pl_PL`) must be filled in `locales/i18n.yaml` (i.e. `shared-libs/crates/start-core/locales/i18n.yaml`), alphabetically ordered within their section. Use `t!()`, match the key namespace to the module path, and use kebab-case for multi-word segments. See `i18n-patterns.md`. Compile-time validation catches missing keys. +- **`#[ts(export)]` changes need a multi-step rebuild.** Use `#[serde(rename_all = "camelCase")]` for JavaScript field names. Add `#[ts(type = "string")]` for unsupported string-backed Rust types and `#[ts(type = "number")]` when a `u64` should not become `bigint`. Editing a `#[ts(export)]` struct/enum in Rust does _not_ update web or container-runtime. From the repo root, run `make start-core-ts-bindings` (regenerates `shared-libs/crates/start-core/bindings/`, then rsyncs into `shared-libs/ts-modules/start-core/lib/osBindings/`) and then rebuild the affected TS lib(s): `cd shared-libs/ts-modules/start-core && make dist` (what web consumes) and/or `cd projects/start-sdk && make bundle` (the SDK bundle container-runtime consumes). See [ARCHITECTURE.md](ARCHITECTURE.md#cross-layer-verification). - **A `///` doc comment on a `#[ts(export)]` type lands in the binding** as a TSDoc block (see `CheckPortV6Res.ts`). Adding or editing one changes the generated `.ts`, so it is not a comment-only change — regenerate. - **A change to the CLI surface needs `make manpages`.** Adding or renaming a subcommand, or editing the `about.*`/`help.arg.*` key behind one, changes the committed man pages under `projects/*/man/` — a new subcommand adds a page _and_ a line to its parent's. The `Generated Artifacts` CI job regenerates them and fails on any drift, in the same job as the TS bindings, so a CLI change and a `#[ts(export)]` change each need their own regeneration step. Note the gate reads `git status`, so staging the result is not enough — commit it. - **Bindings can only come from a compile.** ts-rs emits each binding from an `export_bindings_*` `#[test]` fn the derive macro generates inside this lib, so `make start-core-ts-bindings` necessarily builds the crate; there is no lighter command, and hand-editing a generated file just fails the CI gate. If the build does not fit locally, push the branch instead: the `Generated Artifacts` job regenerates the bindings and the man pages, and a failing gate uploads them as the `generated-artifacts` artifact to download and commit. It is heavier than it needs to be because the root `Cargo.toml` sets `[profile.test] opt-level = 3` — on a constrained machine, cap the container (`--cpus`/`--memory`) and pass `CARGO_BUILD_JOBS`, which doesn't change fingerprints and so reuses a warm `target/`. diff --git a/shared-libs/crates/start-core/ARCHITECTURE.md b/shared-libs/crates/start-core/ARCHITECTURE.md index 7dd6c074b3..a32ad6abad 100644 --- a/shared-libs/crates/start-core/ARCHITECTURE.md +++ b/shared-libs/crates/start-core/ARCHITECTURE.md @@ -98,8 +98,7 @@ Until both steps run, a changed `#[ts(export)]` type is out of sync with everyth ## Further reading - [README.md](README.md) — what this crate is and how to use it -- [CONTRIBUTING.md](CONTRIBUTING.md) — build, test, format, and contribution workflow -- [AGENTS.md](AGENTS.md) — operating rules for AI/dev work (`CLAUDE.md` is a one-line `@AGENTS.md` import) +- [AGENTS.md](AGENTS.md) — build, test, format, and operating rules (`CLAUDE.md` is a one-line `@AGENTS.md` import) - [rpc-toolkit.md](rpc-toolkit.md) — JSON-RPC handler patterns - [patchdb.md](patchdb.md) — Patch-DB watch patterns and TypedDbWatch - [i18n-patterns.md](i18n-patterns.md) — Internationalization conventions diff --git a/shared-libs/crates/start-core/CONTRIBUTING.md b/shared-libs/crates/start-core/CONTRIBUTING.md deleted file mode 100644 index d3d8d9d005..0000000000 --- a/shared-libs/crates/start-core/CONTRIBUTING.md +++ /dev/null @@ -1,80 +0,0 @@ -# Contributing to start-core - -The shared Rust backend lib (`start-core`, lib name `start_core`) at `shared-libs/crates/start-core`. -For general environment setup, cloning, and the monorepo build system, see the repo-root -[CONTRIBUTING.md](../../../CONTRIBUTING.md). - -## Documentation - -This sub-tree's docs split across four files: - -- `README.md` — what this is -- `ARCHITECTURE.md` — how it's built -- `CONTRIBUTING.md` — this file; how to contribute -- `AGENTS.md` — AI/dev operating rules (`CLAUDE.md` is a one-line `@AGENTS.md` import; don't edit it) - -**These docs must be kept up to date.** When you change project structure, conventions, build process, or product context, update the relevant file(s) in the same change — do not defer. - -## Prerequisites - -- [Rust](https://rustup.rs) (nightly for formatting) -- [rust-analyzer](https://rust-analyzer.github.io/) recommended -- [Docker](https://docs.docker.com/get-docker/) (for cross-compilation via `rust-zig-builder` container) - -## Building - -Run from the repo root: - -```bash -cargo check -p start-core # Type check -cargo build -p start-os --bin startbox # Build a product binary -``` - -## Testing - -Run from the repo root: - -```bash -make start-core-test # Run the full suite (wraps run-tests.sh) -cargo test -p start-core --features=test # Run a specific test -``` - -## Formatting - -Run from the repo root: - -```bash -make start-core-format # Format with nightly rustfmt -make start-core-format-check # Read-only check (CI) -``` - -## Adding a New RPC Endpoint - -1. Define a params struct with `#[derive(Deserialize, Serialize)]` -2. Choose a handler type (`from_fn_async` for most cases) -3. Write the handler function: `async fn my_handler(ctx: RpcContext, params: MyParams) -> Result` -4. Register it in the appropriate `ParentHandler` tree -5. If params/response should be available in TypeScript, add `#[derive(TS)]` and `#[ts(export)]` - -See [rpc-toolkit.md](rpc-toolkit.md) for full handler patterns and all four handler types. - -## Adding TS-Exported Types - -When a Rust type needs to be available in TypeScript (for the web frontend or SDK): - -1. Add `ts_rs::TS` to the derive list and `#[ts(export)]` to the struct/enum -2. Use `#[serde(rename_all = "camelCase")]` for JS-friendly field names -3. For types that don't implement TS (like `DateTime`, `exver::Version`), use `#[ts(type = "string")]` overrides -4. For `u64` fields that should be JS `number` (not `bigint`), use `#[ts(type = "number")]` -5. Run `make start-core-ts-bindings` to regenerate — files appear in `shared-libs/crates/start-core/bindings/` then sync to `shared-libs/ts-modules/start-core/lib/osBindings/` -6. Rebuild the affected TS lib(s): `cd shared-libs/ts-modules/start-core && make dist` (web) and/or `cd projects/start-sdk && make bundle` (the SDK bundle for container-runtime) - -## Adding i18n Keys - -1. Add the key to `locales/i18n.yaml` (i.e. `shared-libs/crates/start-core/locales/i18n.yaml`) with all 5 language translations -2. Use the `t!("your.key.name")` macro in Rust code -3. Follow existing namespace conventions — match the module path where the key is used -4. Use kebab-case for multi-word segments -5. Translations are validated at compile time - -See [i18n-patterns.md](i18n-patterns.md) for full conventions. diff --git a/shared-libs/crates/start-core/README.md b/shared-libs/crates/start-core/README.md index fc8338b64f..bd4095cd6f 100644 --- a/shared-libs/crates/start-core/README.md +++ b/shared-libs/crates/start-core/README.md @@ -28,7 +28,7 @@ than enabling `MultiExecutable` entrypoints. ## Requirements -- [Rust](https://rustup.rs) (nightly toolchain for formatting) +- [Rust](https://rustup.rs) (stable; formatting uses a pinned nightly container) - [rust-analyzer](https://rust-analyzer.github.io/) recommended - [Docker](https://docs.docker.com/get-docker/) (for the `rust-zig-builder` cross-compile container used by the test/build scripts) @@ -39,7 +39,7 @@ Run from the repo root (the single Cargo workspace): ```bash cargo check -p start-core # type-check the library make start-core-test # run the test suite (wraps run-tests.sh) -make start-core-format # nightly rustfmt on this crate +make start-core-format # pinned nightly rustfmt for shared Rust crates ``` To build a product binary, build its crate, e.g. `cargo build -p start-os --bin startbox`. @@ -47,8 +47,7 @@ To build a product binary, build its crate, e.g. `cargo build -p start-os --bin ## Documentation - [ARCHITECTURE.md](ARCHITECTURE.md) — module map, RPC pattern, patch-db data flow -- [CONTRIBUTING.md](CONTRIBUTING.md) — build, test, format, and contribution workflow -- [AGENTS.md](AGENTS.md) — operating rules for AI/dev work in this crate +- [AGENTS.md](AGENTS.md) — build, test, format, and operating rules for this crate Topical deep-dives: [rpc-toolkit.md](rpc-toolkit.md), [patchdb.md](patchdb.md), [i18n-patterns.md](i18n-patterns.md), [core-rust-patterns.md](core-rust-patterns.md), diff --git a/shared-libs/crates/start-core/locales/i18n.yaml b/shared-libs/crates/start-core/locales/i18n.yaml index d1ce4f0703..2184baff02 100644 --- a/shared-libs/crates/start-core/locales/i18n.yaml +++ b/shared-libs/crates/start-core/locales/i18n.yaml @@ -304,6 +304,13 @@ setup.restoring-backup: fr_FR: "Restauration de la sauvegarde" pl_PL: "Przywracanie kopii zapasowej" +setup.transfer-source-is-destination: + en_US: "The transfer source and destination must be different drives" + de_DE: "Quell- und Ziellaufwerk der Übertragung müssen unterschiedlich sein" + es_ES: "Las unidades de origen y destino de la transferencia deben ser diferentes" + fr_FR: "Les disques source et destination du transfert doivent être différents" + pl_PL: "Dysk źródłowy i docelowy transferu muszą być różne" + setup.transferring-data: en_US: "Transferring data" de_DE: "Daten werden übertragen" diff --git a/shared-libs/crates/start-core/src/bins/start_init.rs b/shared-libs/crates/start-core/src/bins/start_init.rs index a8b9eed19e..07195e0aac 100644 --- a/shared-libs/crates/start-core/src/bins/start_init.rs +++ b/shared-libs/crates/start-core/src/bins/start_init.rs @@ -6,7 +6,7 @@ use crate::context::rpc::InitRpcContextPhases; use crate::context::{DiagnosticContext, InitContext, RpcContext, SetupContext}; use crate::disk::REPAIR_DISK_PATH; use crate::disk::fsck::RepairStrategy; -use crate::disk::main::{DEFAULT_PASSWORD, Ext4Conversion}; +use crate::disk::main::{DEFAULT_PASSWORD, ImportMode}; use crate::firmware::{check_for_firmware_update, update_firmware}; use crate::init::{InitPhases, STANDBY_MODE_PATH}; use crate::net::gateway::WildcardListener; @@ -149,12 +149,11 @@ async fn setup_or_init( let requires_reboot = crate::disk::main::import( &*disk_guid, DATA_DIR, - if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { + ImportMode::ReadWrite(if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen - }, - Ext4Conversion::Convert, + }), if disk_guid.ends_with("_UNENC") { None } else { diff --git a/shared-libs/crates/start-core/src/disk/fsck/ext4.rs b/shared-libs/crates/start-core/src/disk/fsck/ext4.rs index 53197c080a..3e2a818828 100644 --- a/shared-libs/crates/start-core/src/disk/fsck/ext4.rs +++ b/shared-libs/crates/start-core/src/disk/fsck/ext4.rs @@ -15,7 +15,13 @@ use crate::disk::fsck::RequiresReboot; pub async fn e2fsck_preen( logicalname: impl AsRef + std::fmt::Debug, ) -> Result { - e2fsck_runner(Command::new("e2fsck").arg("-p"), logicalname).await + e2fsck_runner(Command::new("e2fsck").arg("-p"), logicalname, false).await +} + +pub async fn e2fsck_preen_strict( + logicalname: impl AsRef + std::fmt::Debug, +) -> Result { + e2fsck_runner(Command::new("e2fsck").arg("-p"), logicalname, true).await } fn backup_existing_undo_file<'a>(path: &'a Path) -> BoxFuture<'a, Result<(), Error>> { @@ -51,6 +57,7 @@ pub async fn e2fsck_aggressive( e2fsck_runner( Command::new("e2fsck").arg("-y").arg("-z").arg(undo_path), logicalname, + false, ) .await } @@ -58,6 +65,7 @@ pub async fn e2fsck_aggressive( async fn e2fsck_runner( e2fsck_cmd: &mut Command, logicalname: impl AsRef + std::fmt::Debug, + fail_on_uncorrected: bool, ) -> Result { let e2fsck_out = e2fsck_cmd.arg(logicalname.as_ref()).output().await?; let e2fsck_stderr = String::from_utf8(e2fsck_out.stderr)?; @@ -86,6 +94,12 @@ async fn e2fsck_runner( ), ); } + if fail_on_uncorrected && code & 4 != 0 { + return Err(Error::new( + eyre!("{}", t!("disk.fsck.e2fsck-error", stderr = e2fsck_stderr)), + crate::ErrorKind::DiskManagement, + )); + } if code < 8 { if code & 2 != 0 { tracing::warn!("{}", t!("disk.fsck.reboot-required")); diff --git a/shared-libs/crates/start-core/src/disk/main.rs b/shared-libs/crates/start-core/src/disk/main.rs index 23973a07ce..83d8245606 100644 --- a/shared-libs/crates/start-core/src/disk/main.rs +++ b/shared-libs/crates/start-core/src/disk/main.rs @@ -7,10 +7,11 @@ use rust_i18n::t; use tokio::process::Command; use tracing::instrument; +use super::fsck::ext4::e2fsck_preen_strict; use super::fsck::{RepairStrategy, RequiresReboot, detect_filesystem}; use super::util::pvscan; use crate::disk::mount::filesystem::block_dev::BlockDev; -use crate::disk::mount::filesystem::{FileSystem, ReadWrite}; +use crate::disk::mount::filesystem::{FileSystem, ReadOnly, ReadWrite}; use crate::disk::mount::util::unmount; use crate::progress::FullProgressTracker; use crate::util::Invoke; @@ -20,13 +21,15 @@ pub const PASSWORD_PATH: &'static str = "/run/startos/password"; pub const DEFAULT_PASSWORD: &'static str = "password"; pub const MAIN_FS_SIZE: FsSize = FsSize::Gigabytes(8); -/// Whether mounting may convert an ext4 data partition to btrfs in place. -/// Conversion rewrites the partition, so a caller that only reads a drive it is -/// leaving behind must pass `Preserve`. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Ext4Conversion { - Convert, - Preserve, +#[derive(Debug, Clone, Copy)] +pub(crate) enum ImportMode { + ReadOnly, + ReadWrite(RepairStrategy), +} +impl ImportMode { + fn is_read_only(self) -> bool { + matches!(self, Self::ReadOnly) + } } #[instrument(skip_all)] @@ -178,16 +181,59 @@ pub async fn create_all_fs>( Ok(()) } +async fn open_luks( + blockdev_path: &Path, + mapper_name: &str, + password: &str, + read_only: bool, +) -> Result { + let mut cryptsetup = Command::new("cryptsetup"); + cryptsetup.arg("-q").arg("luksOpen"); + if read_only { + cryptsetup.arg("--readonly"); + } else { + cryptsetup.arg("--allow-discards"); + } + let mut key = std::io::Cursor::new(password.as_bytes()); + cryptsetup + .arg("--key-file=-") + .arg(format!("--keyfile-size={}", password.len())) + .arg(blockdev_path) + .arg(mapper_name) + .input(Some(&mut key)) + .invoke(crate::ErrorKind::DiskManagement) + .await?; + Ok(Path::new("/dev/mapper").join(mapper_name)) +} + +async fn close_luks(mapper_name: &str) -> Result<(), Error> { + Command::new("cryptsetup") + .arg("-q") + .arg("luksClose") + .arg(mapper_name) + .invoke(crate::ErrorKind::DiskManagement) + .await?; + Ok(()) +} + +async fn set_block_read_only(blockdev_path: &Path, read_only: bool) -> Result<(), Error> { + Command::new("blockdev") + .arg(if read_only { "--setro" } else { "--setrw" }) + .arg(blockdev_path) + .invoke(crate::ErrorKind::DiskManagement) + .await?; + Ok(()) +} + #[instrument(skip_all)] pub async fn unmount_fs>(guid: &str, datadir: P, name: &str) -> Result<(), Error> { unmount(datadir.as_ref().join(name), false).await?; - if !guid.ends_with("_UNENC") { - Command::new("cryptsetup") - .arg("-q") - .arg("luksClose") - .arg(format!("{}_{}", guid, name)) - .invoke(crate::ErrorKind::DiskManagement) - .await?; + let mapper_name = format!("{guid}_{name}"); + if tokio::fs::metadata(Path::new("/dev/mapper").join(&mapper_name)) + .await + .is_ok() + { + close_luks(&mapper_name).await?; } Ok(()) @@ -220,12 +266,60 @@ pub async fn export>(guid: &str, datadir: P) -> Result<(), Error> Ok(()) } +fn record_cleanup_error(first_error: &mut Option, result: Result<(), Error>) { + if let Err(error) = result { + if first_error.is_none() { + *first_error = Some(error); + } else { + tracing::error!(?error, "Additional disk cleanup error"); + } + } +} + #[instrument(skip_all)] -pub async fn import>( +pub(crate) async fn deactivate>(guid: &str, datadir: P) -> Result<(), Error> { + let mut first_error = None; + for name in ["package-data", "main"] { + record_cleanup_error( + &mut first_error, + unmount(datadir.as_ref().join(name), false).await, + ); + let mapper_name = format!("{guid}_{name}"); + if tokio::fs::metadata(Path::new("/dev/mapper").join(&mapper_name)) + .await + .is_ok() + { + record_cleanup_error(&mut first_error, close_luks(&mapper_name).await); + } + } + record_cleanup_error( + &mut first_error, + Command::new("vgchange") + .arg("-an") + .arg(guid) + .invoke(crate::ErrorKind::DiskManagement) + .await + .map(|_| ()), + ); + record_cleanup_error( + &mut first_error, + Command::new("vgexport") + .arg(guid) + .invoke(crate::ErrorKind::DiskManagement) + .await + .map(|_| ()), + ); + match first_error { + Some(error) => Err(error), + None => Ok(()), + } +} + +#[instrument(skip_all)] +pub(crate) async fn import>( guid: &str, datadir: P, - repair: RepairStrategy, - conversion: Ext4Conversion, + mode: ImportMode, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { @@ -276,48 +370,34 @@ pub async fn import>( .arg(guid) .invoke(crate::ErrorKind::DiskManagement) .await?; - mount_all_fs(guid, datadir, repair, conversion, password, progress).await + mount_all_fs(guid, datadir, mode, password, progress).await } #[instrument(skip_all)] -pub async fn mount_fs>( +pub(crate) async fn mount_fs>( guid: &str, datadir: P, name: &str, - repair: RepairStrategy, - conversion: Ext4Conversion, + mode: ImportMode, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { let orig_path = Path::new("/dev").join(guid).join(name); - let mut blockdev_path = orig_path.clone(); - let full_name = format!("{}_{}", guid, name); - if !guid.ends_with("_UNENC") { - let password = password.unwrap_or(DEFAULT_PASSWORD); - if let Some(parent) = Path::new(PASSWORD_PATH).parent() { - tokio::fs::create_dir_all(parent).await?; - } - tokio::fs::write(PASSWORD_PATH, password) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?; - Command::new("cryptsetup") - .arg("-q") - .arg("luksOpen") - .arg("--allow-discards") - .arg(format!("--key-file={}", PASSWORD_PATH)) - .arg(format!("--keyfile-size={}", password.len())) - .arg(&blockdev_path) - .arg(&full_name) - .invoke(crate::ErrorKind::DiskManagement) - .await?; - tokio::fs::remove_file(PASSWORD_PATH) - .await - .with_ctx(|_| (crate::ErrorKind::Filesystem, PASSWORD_PATH))?; - blockdev_path = Path::new("/dev/mapper").join(&full_name); + let mapper_name = format!("{guid}_{name}"); + let encrypted = !guid.ends_with("_UNENC"); + let password = password.unwrap_or(DEFAULT_PASSWORD); + if mode.is_read_only() { + set_block_read_only(&orig_path, true).await?; } + let mut blockdev_path = if encrypted { + open_luks(&orig_path, &mapper_name, password, mode.is_read_only()).await? + } else { + orig_path.clone() + }; - let fs_type = detect_filesystem(&blockdev_path).await?; - if fs_type == "ext2" && conversion == Ext4Conversion::Convert { + if matches!(mode, ImportMode::ReadWrite(_)) + && detect_filesystem(&blockdev_path).await? == "ext2" + { let mut convert_phase = progress.map(|p| p.add_phase(t!("disk.main.converting-to-btrfs").into(), Some(50))); if let Some(ref mut phase) = convert_phase { @@ -352,7 +432,6 @@ pub async fn mount_fs>( .capture(false) .invoke(ErrorKind::DiskManagement) .await?; - // Delete ext2_saved subvolume and defragment after conversion let tmp_mount = datadir.as_ref().join(format!("{name}.convert-tmp")); tokio::fs::create_dir_all(&tmp_mount).await?; BlockDev::new(&blockdev_path) @@ -377,91 +456,151 @@ pub async fn mount_fs>( } } - let reboot = repair.fsck(&blockdev_path).await?; + let mut reboot = match mode { + ImportMode::ReadOnly => RequiresReboot(false), + ImportMode::ReadWrite(repair) => { + let reboot = repair.fsck(&blockdev_path).await?; + if !guid.ends_with("_UNENC") { + let luks_folder = Path::new("/media/startos/config/luks"); + tokio::fs::create_dir_all(luks_folder).await?; + let tmp_luks_bak = luks_folder.join(format!(".{mapper_name}.luks.bak.tmp")); + if tokio::fs::metadata(&tmp_luks_bak).await.is_ok() { + tokio::fs::remove_file(&tmp_luks_bak).await?; + } + let luks_bak = luks_folder.join(format!("{mapper_name}.luks.bak")); + Command::new("cryptsetup") + .arg("-q") + .arg("luksHeaderBackup") + .arg("--header-backup-file") + .arg(&tmp_luks_bak) + .arg(&orig_path) + .invoke(crate::ErrorKind::DiskManagement) + .await?; + tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?; + } + reboot + } + }; + + let mountpoint = datadir.as_ref().join(name); + match mode { + ImportMode::ReadOnly => { + let initial_error = match BlockDev::new(&blockdev_path) + .mount(&mountpoint, ReadOnly) + .await + { + Ok(()) => return Ok(reboot), + Err(error) => error, + }; + let fs_type = match detect_filesystem(&blockdev_path).await.as_deref() { + Ok(fs_type @ ("ext2" | "btrfs")) => fs_type.to_owned(), + _ => return Err(initial_error), + }; + + if encrypted { + close_luks(&mapper_name).await?; + } + set_block_read_only(&orig_path, false).await?; + if encrypted { + blockdev_path = match open_luks(&orig_path, &mapper_name, password, false).await { + Ok(blockdev_path) => blockdev_path, + Err(error) => { + set_block_read_only(&orig_path, true).await.log_err(); + return Err(error); + } + }; + } - if !guid.ends_with("_UNENC") { - // Backup LUKS header if e2fsck succeeded - let luks_folder = Path::new("/media/startos/config/luks"); - tokio::fs::create_dir_all(luks_folder).await?; - let tmp_luks_bak = luks_folder.join(format!(".{full_name}.luks.bak.tmp")); - if tokio::fs::metadata(&tmp_luks_bak).await.is_ok() { - tokio::fs::remove_file(&tmp_luks_bak).await?; + let recovery_result = match fs_type.as_str() { + "ext2" => e2fsck_preen_strict(&blockdev_path).await, + "btrfs" => { + async { + BlockDev::new(&blockdev_path) + .mount(&mountpoint, ReadOnly) + .await?; + unmount(&mountpoint, false).await?; + Ok(RequiresReboot(false)) + } + .await + } + _ => unreachable!(), + }; + let close_result = if encrypted { + close_luks(&mapper_name).await + } else { + Ok(()) + }; + let protect_result = set_block_read_only(&orig_path, true).await; + let recovery_reboot = match recovery_result { + Ok(reboot) => reboot, + Err(error) => { + close_result.log_err(); + protect_result.log_err(); + return Err(error); + } + }; + close_result?; + protect_result?; + reboot |= recovery_reboot; + + if encrypted { + blockdev_path = open_luks(&orig_path, &mapper_name, password, true).await?; + } + BlockDev::new(&blockdev_path) + .mount(&mountpoint, ReadOnly) + .await?; + } + ImportMode::ReadWrite(_) => { + BlockDev::new(&blockdev_path) + .mount(&mountpoint, ReadWrite) + .await? } - let luks_bak = luks_folder.join(format!("{full_name}.luks.bak")); - Command::new("cryptsetup") - .arg("-q") - .arg("luksHeaderBackup") - .arg("--header-backup-file") - .arg(&tmp_luks_bak) - .arg(&orig_path) - .invoke(crate::ErrorKind::DiskManagement) - .await?; - tokio::fs::rename(&tmp_luks_bak, &luks_bak).await?; } - BlockDev::new(&blockdev_path) - .mount(datadir.as_ref().join(name), ReadWrite) - .await?; - Ok(reboot) } #[instrument(skip_all)] -pub async fn mount_all_fs>( +pub(crate) async fn mount_all_fs>( guid: &str, datadir: P, - repair: RepairStrategy, - conversion: Ext4Conversion, + mode: ImportMode, password: Option<&str>, progress: Option<&FullProgressTracker>, ) -> Result { let mut reboot = RequiresReboot(false); - reboot |= mount_fs( - guid, &datadir, "main", repair, conversion, password, progress, - ) - .await?; - reboot |= mount_fs( - guid, - &datadir, - "package-data", - repair, - conversion, - password, - progress, - ) - .await?; + reboot |= mount_fs(guid, &datadir, "main", mode, password, progress).await?; + reboot |= mount_fs(guid, &datadir, "package-data", mode, password, progress).await?; Ok(reboot) } -/// Temporarily activates a VG and opens LUKS to probe the `package-data` -/// filesystem type. Returns `None` if probing fails (e.g. LV doesn't exist). +/// Probes `package-data` on an inactive or exported volume group. #[instrument(skip_all)] pub async fn probe_package_data_fs(guid: &str) -> Result, Error> { - // If the target block device is already accessible (e.g. this is the - // currently active system VG), probe it directly without any - // import/activate/open/cleanup steps. + let mapper_name = format!("{guid}_package-data"); + let lv_path = Path::new("/dev").join(guid).join("package-data"); let blockdev_path = if !guid.ends_with("_UNENC") { - PathBuf::from(format!("/dev/mapper/{guid}_package-data")) + Path::new("/dev/mapper").join(&mapper_name) } else { - Path::new("/dev").join(guid).join("package-data") + lv_path.clone() }; if tokio::fs::metadata(&blockdev_path).await.is_ok() { return detect_filesystem(&blockdev_path).await.map(Some); } + let was_active = tokio::fs::metadata(&lv_path).await.is_ok(); - // Import and activate the VG - match Command::new("vgimport") + let imported = match Command::new("vgimport") .arg(guid) .invoke(ErrorKind::DiskManagement) .await { - Ok(_) => {} + Ok(_) => true, Err(e) if format!("{}", e.source) .lines() .any(|l| l.trim() == format!("Volume group \"{}\" is not exported", guid)) => { - // Already imported, that's fine + false } Err(e) => { tracing::warn!( @@ -470,13 +609,28 @@ pub async fn probe_package_data_fs(guid: &str) -> Result, Error> ); return Ok(None); } - } + }; if let Err(e) = Command::new("vgchange") .arg("-ay") .arg(guid) .invoke(ErrorKind::DiskManagement) .await { + if !was_active { + Command::new("vgchange") + .arg("-an") + .arg(guid) + .invoke(ErrorKind::DiskManagement) + .await + .log_err(); + } + if imported { + Command::new("vgexport") + .arg(guid) + .invoke(ErrorKind::DiskManagement) + .await + .log_err(); + } tracing::warn!( "{}", t!("disk.main.could-not-activate-vg", guid = guid, error = e) @@ -486,63 +640,40 @@ pub async fn probe_package_data_fs(guid: &str) -> Result, Error> let mut opened_luks = false; let result = async { - let lv_path = Path::new("/dev").join(guid).join("package-data"); if tokio::fs::metadata(&lv_path).await.is_err() { return Ok(None); } let blockdev_path = if !guid.ends_with("_UNENC") { - let full_name = format!("{guid}_package-data"); - let password = DEFAULT_PASSWORD; - if let Some(parent) = Path::new(PASSWORD_PATH).parent() { - tokio::fs::create_dir_all(parent).await?; - } - tokio::fs::write(PASSWORD_PATH, password) - .await - .with_ctx(|_| (ErrorKind::Filesystem, PASSWORD_PATH))?; - Command::new("cryptsetup") - .arg("-q") - .arg("luksOpen") - .arg("--allow-discards") - .arg(format!("--key-file={PASSWORD_PATH}")) - .arg(format!("--keyfile-size={}", password.len())) - .arg(&lv_path) - .arg(&full_name) - .invoke(ErrorKind::DiskManagement) - .await?; - let _ = tokio::fs::remove_file(PASSWORD_PATH).await; + let blockdev_path = open_luks(&lv_path, &mapper_name, DEFAULT_PASSWORD, true).await?; opened_luks = true; - PathBuf::from(format!("/dev/mapper/{full_name}")) + blockdev_path } else { - lv_path.clone() + lv_path }; detect_filesystem(&blockdev_path).await.map(Some) } .await; - // Always clean up: close LUKS, deactivate VG, export VG if opened_luks { - let full_name = format!("{guid}_package-data"); - Command::new("cryptsetup") - .arg("-q") - .arg("luksClose") - .arg(&full_name) + close_luks(&mapper_name).await.log_err(); + } + if !was_active { + Command::new("vgchange") + .arg("-an") + .arg(guid) + .invoke(ErrorKind::DiskManagement) + .await + .log_err(); + } + if imported { + Command::new("vgexport") + .arg(guid) .invoke(ErrorKind::DiskManagement) .await .log_err(); } - Command::new("vgchange") - .arg("-an") - .arg(guid) - .invoke(ErrorKind::DiskManagement) - .await - .log_err(); - Command::new("vgexport") - .arg(guid) - .invoke(ErrorKind::DiskManagement) - .await - .log_err(); result } diff --git a/shared-libs/crates/start-core/src/setup.rs b/shared-libs/crates/start-core/src/setup.rs index 59af6bec7b..a9d146445d 100644 --- a/shared-libs/crates/start-core/src/setup.rs +++ b/shared-libs/crates/start-core/src/setup.rs @@ -28,7 +28,7 @@ use crate::context::{CliContext, RpcContext, SetupContext}; use crate::db::model::Database; use crate::disk::REPAIR_DISK_PATH; use crate::disk::fsck::RepairStrategy; -use crate::disk::main::{DEFAULT_PASSWORD, Ext4Conversion}; +use crate::disk::main::{DEFAULT_PASSWORD, ImportMode}; use crate::disk::mount::filesystem::ReadWrite; use crate::disk::mount::filesystem::cifs::Cifs; use crate::disk::mount::guard::{GenericMountGuard, TmpMountGuard}; @@ -43,7 +43,10 @@ use crate::shutdown::Shutdown; use crate::system::{KeyboardOptions, SetLanguageParams, save_language, sync_kiosk}; use crate::util::Invoke; use crate::util::crypto::EncryptedWire; -use crate::util::io::{Counter, create_file, dir_copy, dir_size, read_file_to_string}; +use crate::util::io::{ + Counter, create_file, dir_copy, dir_copy_excluding, dir_size, dir_size_excluding, + read_file_to_string, +}; use crate::util::serde::{HandlerExtSerde, IoFormat, Pem}; use crate::{DATA_DIR, Error, ErrorKind, MAIN_DATA, PACKAGE_DATA, PLATFORM, ResultExt}; @@ -295,12 +298,11 @@ pub async fn attach( let requires_reboot = crate::disk::main::import( &*disk_guid, DATA_DIR, - if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { + ImportMode::ReadWrite(if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen - }, - Ext4Conversion::Convert, + }), if disk_guid.ends_with("_UNENC") { None } else { @@ -493,8 +495,7 @@ pub async fn setup_data_drive( let _ = crate::disk::main::import( &*guid, DATA_DIR, - RepairStrategy::Preen, - Ext4Conversion::Convert, + ImportMode::ReadWrite(RepairStrategy::Preen), encryption_password, None, ) @@ -762,6 +763,15 @@ pub async fn execute( hostname, }: SetupExecuteParams, ) -> Result { + if let Some(RecoverySource::Migrate { guid: old_guid }) = &recovery_source { + if old_guid.as_str() == >::as_ref(&guid) { + return Err(Error::new( + eyre!("{}", t!("setup.transfer-source-is-destination")), + ErrorKind::InvalidRequest, + )); + } + } + let password = password .map(|p| { p.decrypt(&ctx).ok_or_else(|| { @@ -893,12 +903,11 @@ pub async fn execute_inner( let requires_reboot = crate::disk::main::import( &*guid, DATA_DIR, - if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { + ImportMode::ReadWrite(if tokio::fs::metadata(REPAIR_DISK_PATH).await.is_ok() { RepairStrategy::Aggressive } else { RepairStrategy::Preen - }, - Ext4Conversion::Convert, + }), if guid.ends_with("_UNENC") { None } else { @@ -1086,73 +1095,87 @@ async fn migrate( restore_phase.start(); restore_phase.set_units(Some(ProgressUnits::Bytes)); - // A transfer only reads the source drive, so it stays as the user's fallback. - let _ = crate::disk::main::import( - &old_guid, - "/media/startos/migrate", - RepairStrategy::Preen, - Ext4Conversion::Preserve, - if guid.ends_with("_UNENC") { - None - } else { - Some(DEFAULT_PASSWORD) - }, - Some(&ctx.progress), - ) - .await?; - let main_transfer_args = ("/media/startos/migrate/main/", formatcp!("{MAIN_DATA}/")); - let package_data_transfer_args = ( - "/media/startos/migrate/package-data/", - formatcp!("{PACKAGE_DATA}/"), - ); - - let tmpdir = Path::new(package_data_transfer_args.0).join("tmp"); - crate::util::io::delete_dir(&tmpdir).await?; - - let ordering = std::sync::atomic::Ordering::Relaxed; - - let main_transfer_size = Counter::new(0, ordering); - let package_data_transfer_size = Counter::new(0, ordering); - - let size = tokio::select! { - res = async { - let (main_size, package_data_size) = try_join!( - dir_size(main_transfer_args.0, Some(&main_transfer_size)), - dir_size(package_data_transfer_args.0, Some(&package_data_transfer_size)) - )?; - Ok::<_, Error>(main_size + package_data_size) - } => { res? }, - res = async { - loop { - tokio::time::sleep(Duration::from_secs(1)).await; - restore_phase.set_total(main_transfer_size.load() + package_data_transfer_size.load()); - } - } => res, - }; + let transfer_result = async { + let requires_reboot = crate::disk::main::import( + old_guid, + "/media/startos/migrate", + ImportMode::ReadOnly, + None, + Some(&ctx.progress), + ) + .await?; + if requires_reboot.0 { + return Err(Error::new( + eyre!("{}", t!("setup.disk-errors-corrected-restart-required")), + ErrorKind::DiskManagement, + )); + } - restore_phase.set_total(size); - - let main_transfer_progress = Counter::new(0, ordering); - let package_data_transfer_progress = Counter::new(0, ordering); - - tokio::select! { - res = async { - try_join!( - dir_copy(main_transfer_args.0, main_transfer_args.1, Some(&main_transfer_progress)), - dir_copy(package_data_transfer_args.0, package_data_transfer_args.1, Some(&package_data_transfer_progress)) - )?; - Ok::<_, Error>(()) - } => { res? }, - res = async { - loop { - tokio::time::sleep(Duration::from_secs(1)).await; - restore_phase.set_done(main_transfer_progress.load() + package_data_transfer_progress.load()); - } - } => res, + let main_transfer_args = ("/media/startos/migrate/main/", formatcp!("{MAIN_DATA}/")); + let package_data_transfer_args = ( + "/media/startos/migrate/package-data/", + formatcp!("{PACKAGE_DATA}/"), + ); + let package_data_tmp = Path::new(package_data_transfer_args.0).join("tmp"); + let ordering = std::sync::atomic::Ordering::Relaxed; + let transfer_size = Counter::new(0, ordering); + + let size = tokio::select! { + res = async { + let (main_size, package_data_size) = try_join!( + dir_size(main_transfer_args.0, Some(&transfer_size)), + dir_size_excluding( + package_data_transfer_args.0, + &package_data_tmp, + Some(&transfer_size), + ) + )?; + Ok::<_, Error>(main_size + package_data_size) + } => { res? }, + res = async { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + restore_phase.set_total(transfer_size.load()); + } + } => res, + }; + + restore_phase.set_total(size); + + let transfer_progress = Counter::new(0, ordering); + + tokio::select! { + res = async { + try_join!( + dir_copy(main_transfer_args.0, main_transfer_args.1, Some(&transfer_progress)), + dir_copy_excluding( + package_data_transfer_args.0, + package_data_transfer_args.1, + &package_data_tmp, + Some(&transfer_progress), + ) + )?; + Ok::<_, Error>(()) + } => { res? }, + res = async { + loop { + tokio::time::sleep(Duration::from_secs(1)).await; + restore_phase.set_done(transfer_progress.load()); + } + } => res, + } + Command::new("sync").invoke(ErrorKind::Filesystem).await?; + Ok::<_, Error>(()) } + .await; - crate::disk::main::export(&old_guid, "/media/startos/migrate").await?; + let deactivate_result = crate::disk::main::deactivate(old_guid, "/media/startos/migrate").await; + if let Err(error) = transfer_result { + deactivate_result.log_err(); + return Err(error); + } + deactivate_result?; restore_phase.complete(); let (account, net_ctrl) = setup_init(&ctx, password, kiosk, hostname, init_phases).await?; diff --git a/shared-libs/crates/start-core/src/util/io.rs b/shared-libs/crates/start-core/src/util/io.rs index c81e63d106..d1a3aed5bc 100644 --- a/shared-libs/crates/start-core/src/util/io.rs +++ b/shared-libs/crates/start-core/src/util/io.rs @@ -243,13 +243,33 @@ pub async fn copy_and_shutdown( Ok(()) } -pub fn dir_size<'a, P: AsRef + 'a + Send + Sync>( +pub fn dir_size<'a, P: AsRef>( path: P, ctr: Option<&'a Counter>, +) -> BoxFuture<'a, Result> { + dir_size_inner(path.as_ref().to_owned(), None, ctr) +} + +pub(crate) fn dir_size_excluding<'a, P: AsRef>( + path: P, + excluded: &'a Path, + ctr: Option<&'a Counter>, +) -> BoxFuture<'a, Result> { + dir_size_inner(path.as_ref().to_owned(), Some(excluded), ctr) +} + +fn dir_size_inner<'a>( + path: PathBuf, + excluded: Option<&'a Path>, + ctr: Option<&'a Counter>, ) -> BoxFuture<'a, Result> { async move { - tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(path.as_ref()).await?) + tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(&path).await?) .try_fold(0, |acc, e| async move { + let path = e.path(); + if excluded.is_some_and(|excluded| excluded == path) { + return Ok(acc); + } let m = e.metadata().await?; Ok(acc + if m.is_file() { @@ -258,7 +278,7 @@ pub fn dir_size<'a, P: AsRef + 'a + Send + Sync>( } m.len() } else if m.is_dir() { - dir_size(e.path(), ctr).await? + dir_size_inner(path, excluded, ctr).await? } else { 0 }) @@ -658,14 +678,37 @@ impl<'a, R: AsyncRead> AsyncRead for CountingReader<'a, R> { } } -pub fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + Send + Sync>( +pub fn dir_copy<'a, P0: AsRef, P1: AsRef>( src: P0, dst: P1, ctr: Option<&'a Counter>, +) -> BoxFuture<'a, Result<(), crate::Error>> { + dir_copy_inner(src.as_ref().to_owned(), dst.as_ref().to_owned(), None, ctr) +} + +pub(crate) fn dir_copy_excluding<'a, P0: AsRef, P1: AsRef>( + src: P0, + dst: P1, + excluded: &'a Path, + ctr: Option<&'a Counter>, +) -> BoxFuture<'a, Result<(), crate::Error>> { + dir_copy_inner( + src.as_ref().to_owned(), + dst.as_ref().to_owned(), + Some(excluded), + ctr, + ) +} + +fn dir_copy_inner<'a>( + src: PathBuf, + dst: PathBuf, + excluded: Option<&'a Path>, + ctr: Option<&'a Counter>, ) -> BoxFuture<'a, Result<(), crate::Error>> { async move { let m = tokio::fs::metadata(&src).await?; - let dst_path = dst.as_ref(); + let dst_path = dst.as_path(); tokio::fs::create_dir_all(&dst_path).await.with_ctx(|_| { ( crate::ErrorKind::Filesystem, @@ -696,11 +739,14 @@ pub fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + S format!("chown {}", dst_path.display()), ) })?; - tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(src.as_ref()).await?) + tokio_stream::wrappers::ReadDirStream::new(tokio::fs::read_dir(&src).await?) .map_err(|e| crate::Error::new(e, crate::ErrorKind::Filesystem)) .try_for_each(|e| async move { - let m = e.metadata().await?; let src_path = e.path(); + if excluded.is_some_and(|excluded| excluded == src_path) { + return Ok(()); + } + let m = e.metadata().await?; let dst_path = dst_path.join(e.file_name()); if m.is_file() { let mut dst_file = create_file(&dst_path).await.with_ctx(|_| { @@ -747,7 +793,7 @@ pub fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + S ) })?; } else if m.is_dir() { - dir_copy(src_path, dst_path, ctr).await?; + dir_copy_inner(src_path, dst_path, excluded, ctr).await?; } else if m.file_type().is_symlink() { tokio::fs::symlink( tokio::fs::read_link(&src_path).await.with_ctx(|_| { @@ -765,7 +811,6 @@ pub fn dir_copy<'a, P0: AsRef + 'a + Send + Sync, P1: AsRef + 'a + S format!("cp -P {} -> {}", src_path.display(), dst_path.display()), ) })?; - // Do not set permissions (see https://unix.stackexchange.com/questions/87200/change-permissions-for-a-symbolic-link) } Ok(()) }) @@ -1839,6 +1884,48 @@ impl Drop for AtomicFile { mod test { use super::*; + #[tokio::test] + async fn directory_helpers_exclude_source_path() { + let root = PathBuf::from(format!( + "/tmp/dir-copy-test-{}-{}", + std::process::id(), + rand::random::() + )); + let src = root.join("src"); + let dst = root.join("dst"); + let excluded = src.join("tmp"); + tokio::fs::create_dir_all(&excluded).await.unwrap(); + tokio::fs::write(src.join("keep"), b"keep").await.unwrap(); + tokio::fs::write(excluded.join("skip"), b"skip") + .await + .unwrap(); + + let size = Counter::new(0, std::sync::atomic::Ordering::Relaxed); + assert_eq!( + dir_size_excluding(&src, &excluded, Some(&size)) + .await + .unwrap(), + 4 + ); + assert_eq!(size.load(), 4); + + let copied = Counter::new(0, std::sync::atomic::Ordering::Relaxed); + dir_copy_excluding(&src, &dst, &excluded, Some(&copied)) + .await + .unwrap(); + assert_eq!(tokio::fs::read(dst.join("keep")).await.unwrap(), b"keep"); + assert_eq!(copied.load(), 4); + assert_eq!( + tokio::fs::metadata(dst.join("tmp")) + .await + .unwrap_err() + .kind(), + std::io::ErrorKind::NotFound + ); + + tokio::fs::remove_dir_all(root).await.unwrap(); + } + #[tokio::test] async fn canonicalize_folds_parent_components_in_missing_tails() { let tmp = PathBuf::from(format!("/tmp/canonicalize-test-{}", std::process::id()));