diff --git a/README.md b/README.md index 18a7e9e..3ec6ebf 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ rpmoci is also similar to a smaller [`rpm-ostree compose image`](https://coreos. rpmoci has a runtime dependency on dnf, so requires a Linux distribution with dnf support. -rpmoci is available to download from crates.io, so you'll need a Rust toolchain. You also need to install the sqlite, python3 and openssl development packages (e.g `sqlite-devel`, `python3-devel` and `openssl-devel` on Fedora and RHEL derivatives). +rpmoci is available to download from crates.io, so you'll need a Rust toolchain. You also need to install the sqlite, python3 and openssl development packages (e.g `sqlite-devel`, `python3-devel` and `openssl-devel` on Fedora and RHEL derivatives). Then install rpmoci via cargo: ```bash @@ -26,7 +26,7 @@ cargo install rpmoci ``` ## Building -Per the above, you'll need dnf, Rust, python3-devel and openssl-devel installed. +Per the above, you'll need dnf, Rust, `sqlite-devel`, `python3-devel`, and `openssl-devel` installed. ```bash cargo build @@ -260,7 +260,7 @@ Adding bzip2-libs 1.0.8-1.cm2 rpmoci can produce bitwise reproducible container image builds, assuming that the RPMs can be reproducibly installed (an rpmoci build won't be reproducible if it involves RPMs that have unreproducible post-install scripts for example). rpmoci attempts to remove sources of non-determinism from the container image, and respects the [SOURCE_DATE_EPOCH](https://reproducible-builds.org/docs/source-date-epoch/) environment variable. -When SOURCE_DATE_EPOCH is not set, the image creation time in the OCI image config is set to the current time. In this scenario rpmoci still removes non-deteministic data from the image, and the build can later be reproduced by setting SOURCE_DATE_EPOCH to the creation time of the image (by converting the timestamp in the image config to seconds since unix epoch). +When SOURCE_DATE_EPOCH is not set, the image creation time in the OCI image config is set to the current time. In this scenario rpmoci still removes non-deteministic data from the image, and the build can later be reproduced by setting SOURCE_DATE_EPOCH to the creation time of the image (by converting the timestamp in the image config to seconds since unix epoch). This feature is only been tested on Mariner Linux, but should work when rpmoci is run on any Linux distribution that writes the rpmdb as a sqlite database to `/var/lib/rpm/rpmdb.sqlite`. @@ -302,5 +302,5 @@ cargo generate-rpm The tests are run via `cargo test`. The integration tests in `tests/it.rs` run `rpmoci build`, so must be run either as root, or with user namespace support setup. -The tests use [test-temp-dir](https://docs.rs/crate/test-temp-dir/latest), so you can set +The tests use [test-temp-dir](https://docs.rs/crate/test-temp-dir/latest), so you can set the `TEST_TEMP_RETAIN` environment variable to `1` so that the test directories are kept around for debugging in `/tests`. diff --git a/src/cli.rs b/src/cli.rs index 54905aa..c53a4d7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -56,6 +56,17 @@ pub enum Command { /// local RPMs being present, which may be useful in dependency updating scenarios. #[clap(long = "from-lockfile")] from_lockfile: bool, + /// Update only the specified package(s), by name. All other packages in the + /// lock file remain pinned to their current versions. Requires an existing, + /// up-to-date lock file. May be specified multiple times. + /// + /// A package may fail to update if another (still-pinned) package has a + /// strict version-lock on it - for example RPMs built from the same source + /// often require each other at the exact same version (e.g. `glibc`, + /// `glibc-common`, `glibc-langpack-en`). In that case, pass all the + /// related packages together: `-p glibc -p glibc-common ...`. + #[clap(short = 'p', long = "package", conflicts_with = "from_lockfile")] + package: Vec, }, /// Build an OCI image Build { diff --git a/src/lib.rs b/src/lib.rs index 09eff88..397f60b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,10 +60,29 @@ pub fn main(command: Command) -> anyhow::Result<()> { Command::Update { manifest_path, from_lockfile, + package, } => { let (cfg, lockfile_path, existing_lockfile) = load_config_and_lock_file(manifest_path)?; - let lockfile = if let Ok(Some(lockfile)) = &existing_lockfile { + let lockfile = if !package.is_empty() { + // `--package` requires an existing, parseable, up-to-date lockfile. + match &existing_lockfile { + Ok(Some(lockfile)) + if lockfile.is_compatible_excluding_local_rpms(&cfg) => + { + lockfile.resolve_updating(&cfg, &package)? + } + Ok(Some(_)) => bail!( + "the lock file is not up-to-date. Use of --package requires that the lock file is up-to-date; run `rpmoci update` first" + ), + Ok(None) => bail!( + "no lock file found. Use of --package requires an existing lock file" + ), + Err(_) => bail!( + "failed to parse existing lock file. Use of --package requires a valid lock file" + ), + } + } else if let Ok(Some(lockfile)) = &existing_lockfile { if lockfile.is_compatible_excluding_local_rpms(&cfg) && from_lockfile { lockfile.resolve_from_previous(&cfg)? } else { diff --git a/src/lockfile/resolve.rs b/src/lockfile/resolve.rs index 43db26b..2e5dd5a 100644 --- a/src/lockfile/resolve.rs +++ b/src/lockfile/resolve.rs @@ -157,6 +157,142 @@ impl Lockfile { lockfile.pkg_specs.clone_from(&cfg.contents.packages); Ok(lockfile) } + + /// Create a lockfile by re-resolving, but pinning every currently locked + /// package to its exact NEVRA *except* those whose names appear in + /// `update`. This is the analogue of `cargo update -p `. + pub fn resolve_updating(&self, cfg: &Config, update: &[String]) -> Result { + let specs = self.build_update_specs(cfg, update)?; + + let mut lockfile = Self::resolve( + specs, + &cfg.contents.repositories, + cfg.contents.gpgkeys.clone(), + cfg.contents.os_release, + )?; + + // For any package whose (name, evr, arch) is unchanged, re-use the + // entry from the previous lockfile verbatim. This prevents spurious + // repoid/checksum churn when the same NEVRA is available from + // multiple repositories (hawkey may tie-break differently between + // runs when given a NEVRA pin spec). + let old_by_key: std::collections::HashMap<_, _> = self + .packages + .iter() + .map(|p| ((p.name.as_str(), p.evr.as_str(), p.arch.as_deref()), p)) + .collect(); + lockfile.packages = lockfile + .packages + .into_iter() + .map(|pkg| { + let key = (pkg.name.as_str(), pkg.evr.as_str(), pkg.arch.as_deref()); + match old_by_key.get(&key) { + Some(old) => (*old).clone(), + None => pkg, + } + }) + .collect(); + + // Merge the previous repo_gpg_config into the new one, preferring + // the previous entries (which correspond to the preserved packages), + // then keep only entries for repos actually referenced by the final + // package set. + let referenced_repos: std::collections::BTreeSet<&str> = lockfile + .packages + .iter() + .map(|p| p.repoid.as_str()) + .collect(); + for (repo, info) in &self.repo_gpg_config { + lockfile + .repo_gpg_config + .entry(repo.clone()) + .or_insert_with(|| info.clone()); + } + lockfile + .repo_gpg_config + .retain(|k, _| referenced_repos.contains(k.as_str())); + + // Preserve the original local packages and manifest-visible pkg_specs + // so the lockfile's compatibility check still passes. + lockfile.local_packages.clone_from(&self.local_packages); + lockfile.pkg_specs.clone_from(&cfg.contents.packages); + Ok(lockfile) + } + + /// Validate a `--package` request and construct the spec list that + /// should be passed to dnf. Extracted for unit testing. + fn build_update_specs(&self, cfg: &Config, update: &[String]) -> Result> { + use std::collections::HashSet; + + // Reject requests to "update" a local RPM - those aren't version-resolved. + let local_names: HashSet<&str> = + self.local_packages.iter().map(|p| p.name.as_str()).collect(); + for name in update { + if local_names.contains(name.as_str()) { + anyhow::bail!( + "`{name}` is a local RPM and cannot be updated via --package" + ); + } + } + + // Validate every requested name actually exists in the lock file. + let locked_names: HashSet<&str> = + self.packages.iter().map(|p| p.name.as_str()).collect(); + for name in update { + if !locked_names.contains(name.as_str()) { + anyhow::bail!("package `{name}` is not in the lock file"); + } + } + + let update: HashSet<&str> = update.iter().map(String::as_str).collect(); + + // Names of packages we are pinning (everything in the lockfile except + // the ones being updated). Loose top-level specs for these names must + // be dropped or DNF sees a conflict between the pin and the loose + // spec (e.g. "libpq" resolving to latest vs "libpq-13.20" pinned). + let pinned_names: HashSet<&str> = self + .packages + .iter() + .map(|p| p.name.as_str()) + .filter(|n| !update.contains(n)) + .collect(); + + // Top-level specs from the manifest (skip local `.rpm` globs; their + // dependencies come from `local_packages.requires`, matching how + // `resolve_from_previous` handles them). Also skip any spec whose + // leading token matches a pinned package name. + let top_level = cfg + .contents + .packages + .iter() + .filter(|s| !s.ends_with(".rpm")) + .filter(|s| { + let name = s.split_whitespace().next().unwrap_or(s.as_str()); + !pinned_names.contains(name) + }) + .cloned(); + + // Requires from local RPMs (same filtering as resolve_from_previous). + let local_requires = self + .local_packages + .iter() + .flat_map(|pkg| pkg.requires.clone()) + .filter(|r| !r.starts_with("rpmlib(")); + + // For every locked package NOT being updated, build a pin spec + // "name-evr.arch" (falling back to "name-evr" if the lockfile predates + // the arch field). dnf/hawkey treats these as exact-version installs. + let pins = self + .packages + .iter() + .filter(|pkg| !update.contains(pkg.name.as_str())) + .map(|pkg| match &pkg.arch { + Some(arch) => format!("{}-{}.{}", pkg.name, pkg.evr, arch), + None => format!("{}-{}", pkg.name, pkg.evr), + }); + + Ok(top_level.chain(local_requires).chain(pins).collect()) + } } /// A wrapper around the dnf.Base object which ensures that plugins are unloaded @@ -435,4 +571,159 @@ mod tests { .unwrap(); assert!(!lock.packages.iter().any(|p| p.name == "pcre2-doc")); } + + /// Fixture lockfile used by the --package unit tests below. + /// Contains three remote packages and one local RPM reference, + /// none of which require network access to parse. + fn fixture_lockfile() -> Lockfile { + let toml = r#" +pkg_specs = ["libpq", "libzstd", "mcl", "./output/myapp-1.0.0.rpm"] + +[[packages]] +name = "libpq" +evr = "13.20-1.el9_5" +repoid = "sas-rpms-escrow" +arch = "x86_64" +[packages.checksum] +algorithm = "sha256" +checksum = "aa" + +[[packages]] +name = "libzstd" +evr = "1.5.1-2.el9" +repoid = "ubi-rpms-virtual" +arch = "x86_64" +[packages.checksum] +algorithm = "sha256" +checksum = "bb" + +[[packages]] +name = "mcl" +evr = "17.2.5-1.el9" +repoid = "alianza" +arch = "x86_64" +[packages.checksum] +algorithm = "sha256" +checksum = "cc" + +[[local_packages]] +name = "myapp" +requires = ["libpq.so.5()(64bit)", "libzstd.so.1()(64bit)", "rpmlib(PayloadFilesHavePrefix)"] +"#; + toml::from_str(toml).expect("fixture lockfile must parse") + } + + fn fixture_config(packages: &[&str]) -> crate::config::Config { + let mut cfg = crate::config::Config::default(); + cfg.contents.packages = packages.iter().map(|s| (*s).to_string()).collect(); + cfg + } + + #[test] + fn update_rejects_unknown_package() { + let lock = fixture_lockfile(); + let cfg = fixture_config(&["libpq", "libzstd", "mcl"]); + let err = lock + .build_update_specs(&cfg, &["does-not-exist".to_string()]) + .unwrap_err(); + assert!( + err.to_string().contains("is not in the lock file"), + "unexpected error: {err}" + ); + } + + #[test] + fn update_rejects_local_rpm_name() { + let lock = fixture_lockfile(); + let cfg = fixture_config(&["libpq", "libzstd", "mcl"]); + let err = lock + .build_update_specs(&cfg, &["myapp".to_string()]) + .unwrap_err(); + assert!( + err.to_string().contains("local RPM"), + "unexpected error: {err}" + ); + } + + #[test] + fn update_single_package_pins_others_and_leaves_target_loose() { + let lock = fixture_lockfile(); + let cfg = fixture_config(&[ + "libpq", + "libzstd", + "mcl", + "./output/myapp-1.0.0.rpm", + ]); + let specs = lock + .build_update_specs(&cfg, &["libzstd".to_string()]) + .unwrap(); + + // The target's loose name must survive so it can float. + assert!(specs.contains(&"libzstd".to_string())); + // Loose top-level specs for pinned packages must be filtered out. + assert!(!specs.contains(&"libpq".to_string())); + assert!(!specs.contains(&"mcl".to_string())); + // Local .rpm globs are never passed through as top-level specs. + assert!(!specs.iter().any(|s| s.ends_with(".rpm"))); + // Every non-updated locked package must appear as an exact NEVRA pin. + assert!(specs.contains(&"libpq-13.20-1.el9_5.x86_64".to_string())); + assert!(specs.contains(&"mcl-17.2.5-1.el9.x86_64".to_string())); + // The target must NOT be pinned. + assert!(!specs.iter().any(|s| s.starts_with("libzstd-1.5.1"))); + // Local package soname requires propagate through. + assert!(specs.iter().any(|s| s == "libpq.so.5()(64bit)")); + // rpmlib() requirements are filtered out. + assert!(!specs.iter().any(|s| s.starts_with("rpmlib("))); + } + + #[test] + fn update_multiple_packages_leaves_all_targets_loose() { + let lock = fixture_lockfile(); + let cfg = fixture_config(&["libpq", "libzstd", "mcl"]); + let specs = lock + .build_update_specs(&cfg, &["libpq".to_string(), "mcl".to_string()]) + .unwrap(); + + // Both targets stay loose. + assert!(specs.contains(&"libpq".to_string())); + assert!(specs.contains(&"mcl".to_string())); + // Neither target is pinned. + assert!(!specs.iter().any(|s| s.starts_with("libpq-13.20"))); + assert!(!specs.iter().any(|s| s.starts_with("mcl-17.2.5"))); + // The one non-target package is pinned and has no loose spec. + assert!(specs.contains(&"libzstd-1.5.1-2.el9.x86_64".to_string())); + let libzstd_loose_count = specs.iter().filter(|s| s.as_str() == "libzstd").count(); + assert_eq!(libzstd_loose_count, 0); + } + + #[test] + fn update_handles_missing_arch_in_old_lockfiles() { + // Older lockfiles lacked the arch field; the pin spec must still work. + let toml = r#" +pkg_specs = ["foo"] + +[[packages]] +name = "foo" +evr = "1.0-1" +repoid = "somerepo" +[packages.checksum] +algorithm = "sha256" +checksum = "dd" + +[[packages]] +name = "bar" +evr = "2.0-1" +repoid = "somerepo" +[packages.checksum] +algorithm = "sha256" +checksum = "ee" +"#; + let lock: Lockfile = toml::from_str(toml).unwrap(); + let cfg = fixture_config(&["foo", "bar"]); + let specs = lock + .build_update_specs(&cfg, &["foo".to_string()]) + .unwrap(); + // bar must be pinned without an arch suffix. + assert!(specs.contains(&"bar-2.0-1".to_string())); + } } diff --git a/tests/it.rs b/tests/it.rs index ac0d1df..67f85b9 100644 --- a/tests/it.rs +++ b/tests/it.rs @@ -93,6 +93,94 @@ fn test_updatable_lockfile() { assert!(!stderr.contains("Removing")); } +/// `update -p ` should update only the named package and leave the +/// other locked entries untouched. +#[test] +fn test_update_single_package() { + let (_tmp_dir, root) = setup_test("updatable_lockfile"); + let output = rpmoci() + .arg("update") + .arg("-p") + .arg("filesystem") + .current_dir(&root) + .env("NO_COLOR", "YES") + .output() + .unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + eprintln!("stderr: {stderr}"); + assert!(output.status.success()); + // Target package was updated. + assert!(stderr.contains("Updating filesystem 1.1-9.cm2 -> ")); + // Non-targeted packages remained pinned at their stale versions. + assert!(!stderr.contains("Updating etcd")); + assert!(!stderr.contains("Updating glibc")); + assert!(!stderr.contains("Removing")); +} + +/// `update -p -p ` should update exactly the named set. +#[test] +fn test_update_multiple_packages() { + let (_tmp_dir, root) = setup_test("updatable_lockfile"); + let output = rpmoci() + .arg("update") + .arg("-p") + .arg("filesystem") + .arg("-p") + .arg("glibc") + .current_dir(&root) + .env("NO_COLOR", "YES") + .output() + .unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + eprintln!("stderr: {stderr}"); + assert!(output.status.success()); + // Both target packages were updated. + assert!(stderr.contains("Updating filesystem 1.1-9.cm2 -> ")); + assert!(stderr.contains("Updating glibc 2.35-1.cm2 -> ")); + // The non-targeted package remained pinned. + assert!(!stderr.contains("Updating etcd")); + assert!(!stderr.contains("Removing")); +} + +/// `update -p ` should fail with a clear error before contacting +/// any repositories. +#[test] +fn test_update_package_not_in_lockfile() { + let (_tmp_dir, root) = setup_test("updatable_lockfile"); + let output = rpmoci() + .arg("update") + .arg("-p") + .arg("definitely-not-a-real-package") + .current_dir(&root) + .env("NO_COLOR", "YES") + .output() + .unwrap(); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + eprintln!("stderr: {stderr}"); + assert!(!output.status.success()); + assert!(stderr.contains("is not in the lock file")); +} + +/// `update --package` combined with `--from-lockfile` should be rejected +/// by clap before any work is done. +#[test] +fn test_update_package_conflicts_with_from_lockfile() { + let (_tmp_dir, root) = setup_test("updatable_lockfile"); + let output = rpmoci() + .arg("update") + .arg("-p") + .arg("filesystem") + .arg("--from-lockfile") + .current_dir(&root) + .env("NO_COLOR", "YES") + .output() + .unwrap(); + assert!(!output.status.success()); + let stderr = std::str::from_utf8(&output.stderr).unwrap(); + eprintln!("stderr: {stderr}"); + assert!(stderr.contains("cannot be used with")); +} + #[test] fn test_unparseable_lockfile() { let (_tmp_dir, root) = setup_test("unparseable_lockfile");