diff --git a/Cargo.lock b/Cargo.lock index 7937ce2704..8751cfac0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3661,6 +3661,14 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-peekable" +version = "0.44.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "serde_cbor" version = "0.11.2" @@ -4239,6 +4247,7 @@ dependencies = [ "spk-cmd-render", "spk-cmd-repo", "spk-cmd-test", + "spk-cmd-workspace", "spk-exec", "spk-schema", "spk-solve", @@ -4646,6 +4655,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "spk-cmd-workspace" +version = "0.44.0" +dependencies = [ + "async-trait", + "clap 4.5.47", + "colored", + "itertools 0.14.0", + "miette", + "spfs", + "spk-build", + "spk-cli-common", + "spk-cmd-make-binary", + "spk-cmd-make-source", + "spk-schema", + "spk-solve", + "spk-storage", + "spk-workspace", + "tracing", +] + [[package]] name = "spk-config" version = "0.44.0" @@ -4702,6 +4732,7 @@ dependencies = [ name = "spk-schema" version = "0.44.0" dependencies = [ + "bracoxide", "config", "data-encoding", "dunce", @@ -4714,12 +4745,14 @@ dependencies = [ "miette", "nom", "nom-supreme", + "once_cell", "proptest", "regex", "relative-path", "ring", "rstest", "serde", + "serde-peekable", "serde_json", "serde_yaml 0.9.34+deprecated", "shellexpand", @@ -4732,6 +4765,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tracing", + "url", "variantly", ] @@ -4974,6 +5008,7 @@ dependencies = [ "serde_yaml 0.9.34+deprecated", "spfs", "spk-schema", + "spk-workspace", "sys-info", "tar", "tempfile", @@ -5001,7 +5036,6 @@ dependencies = [ "serde_json", "serde_yaml 0.9.34+deprecated", "spk-schema", - "spk-solve", "tempfile", "thiserror 1.0.69", "tracing", diff --git a/Cargo.toml b/Cargo.toml index e5c9c9fc1c..390c52205d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ sentry-miette = { version = "0.1.0", path = "crates/sentry-miette" } sentry-tracing = { version = "0.34" } serde = "1.0" serde_json = "1.0" +serde-peekable = { path = "crates/serde-peekable" } serde_yaml = "0.9.25" serial_test = "3.1" shellexpand = "3.1.0" @@ -134,6 +135,7 @@ spk-cmd-make-source = { path = "crates/spk-cli/cmd-make-source" } spk-cmd-render = { path = "crates/spk-cli/cmd-render" } spk-cmd-repo = { path = "crates/spk-cli/cmd-repo" } spk-cmd-test = { path = "crates/spk-cli/cmd-test" } +spk-cmd-workspace = { path = "crates/spk-cli/cmd-workspace" } spk-config = { path = "crates/spk-config" } spk-exec = { path = "crates/spk-exec" } spk-schema = { path = "crates/spk-schema" } @@ -165,6 +167,7 @@ tracing-capture = "0.1" tracing-subscriber = "=0.3.19" ulid = "1.0" unix_mode = "0.1.3" +url = { version = "2.2" } variantly = "0.4" whoami = "1.5" windows = "0.51" diff --git a/crates/serde-peekable/Cargo.toml b/crates/serde-peekable/Cargo.toml new file mode 100644 index 0000000000..d15e11e330 --- /dev/null +++ b/crates/serde-peekable/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "serde-peekable" +version = { workspace = true } +edition = { workspace = true } +authors = { workspace = true } + +[dependencies] +serde = { version = "1.0" } + +[dev-dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" diff --git a/crates/serde-peekable/src/lib.rs b/crates/serde-peekable/src/lib.rs new file mode 100644 index 0000000000..2545bd3e3d --- /dev/null +++ b/crates/serde-peekable/src/lib.rs @@ -0,0 +1,365 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk +// +// Originally sourced from: +// https://gist.github.com/giuseppe998e/0b4f7d92de772e081a90b8003c986110 + +//! Provides peeking into serde deserialization primitives, for use in [`serde::de::Deserialize`] implementations. + +#![warn(missing_docs)] + +use std::marker::PhantomData; + +use serde::__private::de as private_de; +use serde::de; + +/// Wraps around a Serde's MapAccess, providing the ability +/// to peek at the next key and/or value without consuming it. +pub struct PeekableMapAccess<'de, A> { + map: A, + peeked_key: Option>>, + peeked_value: Option>, +} + +impl<'de, A> PeekableMapAccess<'de, A> +where + A: de::MapAccess<'de>, +{ + /// Peeks at the next key in the map without consuming it. + pub fn peek_key_seed(&mut self, seed: K) -> Result, A::Error> + where + K: de::DeserializeSeed<'de>, + { + let key_ref = match self.peeked_key.as_ref() { + Some(key_ref) => key_ref, + None => { + self.peeked_key = Some(self.map.next_key::>()?); + self.peeked_value = None; // Clears the previous peeked value + + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + unsafe { self.peeked_key.as_ref().unwrap_unchecked() } + } + }; + + match key_ref { + Some(key_ref) => { + let deserializer = private_de::ContentRefDeserializer::new(key_ref); + seed.deserialize(deserializer).map(Some) + } + None => Ok(None), + } + } + + /// Peeks at the next value in the map without consuming it. + fn peek_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + let value_ref = match self.peeked_value.as_ref() { + Some(value_ref) => value_ref, + None => { + self.peeked_value = Some(self.map.next_value::>()?); + + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + unsafe { self.peeked_value.as_ref().unwrap_unchecked() } + } + }; + + let deserializer = private_de::ContentRefDeserializer::new(value_ref); + seed.deserialize(deserializer) + } + + /// Peeks at the next key in the map without consuming it. + /// + /// This method exists as a convenience for `Deserialize` implementations. + #[inline] + pub fn peek_key(&mut self) -> Result, A::Error> + where + K: de::Deserialize<'de>, + { + self.peek_key_seed(PhantomData) + } + + /// Peeks at the next value in the map without consuming it. + /// + /// This method exists as a convenience for `Deserialize` implementations. + #[inline] + pub fn peek_value(&mut self) -> Result + where + V: de::Deserialize<'de>, + { + self.peek_value_seed(PhantomData) + } +} + +impl<'de, A> de::MapAccess<'de> for PeekableMapAccess<'de, A> +where + A: de::MapAccess<'de>, +{ + type Error = A::Error; + + fn next_key_seed(&mut self, seed: K) -> Result, Self::Error> + where + K: de::DeserializeSeed<'de>, + { + match self.peeked_key.take() { + Some(Some(key)) => { + let deserializer = private_de::ContentDeserializer::new(key); + seed.deserialize(deserializer).map(Some) + } + Some(None) => Ok(None), + None => self.map.next_key_seed(seed), + } + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: de::DeserializeSeed<'de>, + { + self.peeked_key = None; // Clears the previous peeked key + + match self.peeked_value.take() { + Some(value) => { + let deserializer = private_de::ContentDeserializer::new(value); + seed.deserialize(deserializer) + } + None => self.map.next_value_seed(seed), + } + } + + fn size_hint(&self) -> Option { + self.map.size_hint() + } +} + +impl<'de, A> From for PeekableMapAccess<'de, A> +where + A: de::MapAccess<'de>, +{ + fn from(map: A) -> Self { + Self { + map, + peeked_key: None, + peeked_value: None, + } + } +} + +/// Wraps around a Serde's SeqAccess, providing the ability +/// to peek at the next element without consuming it. +pub struct PeekableSeqAccess<'de, S> { + seq: S, + peeked: Option>>, +} + +impl<'de, S> PeekableSeqAccess<'de, S> +where + S: de::SeqAccess<'de>, +{ + /// Peeks at the next element in the sequence without consuming it. + pub fn peek_element_seed(&mut self, seed: T) -> Result, S::Error> + where + T: de::DeserializeSeed<'de>, + { + let elem_ref = match self.peeked.as_ref() { + Some(elem_ref) => elem_ref, + None => { + self.peeked = Some(self.seq.next_element::>()?); + + // SAFETY: a `None` variant for `self` would have been replaced by a `Some` + // variant in the code above. + unsafe { self.peeked.as_ref().unwrap_unchecked() } + } + }; + + match elem_ref { + Some(elem_ref) => { + let deserializer = private_de::ContentRefDeserializer::new(elem_ref); + seed.deserialize(deserializer).map(Some) + } + None => Ok(None), + } + } + + /// Peeks at the next element in the sequence without consuming it. + /// + /// This method exists as a convenience for `Deserialize` implementations. + #[inline] + pub fn peek_element(&mut self) -> Result, S::Error> + where + T: de::Deserialize<'de>, + { + self.peek_element_seed(PhantomData) + } +} + +impl<'de, S> de::SeqAccess<'de> for PeekableSeqAccess<'de, S> +where + S: de::SeqAccess<'de>, +{ + type Error = S::Error; + + fn next_element_seed(&mut self, seed: T) -> Result, Self::Error> + where + T: de::DeserializeSeed<'de>, + { + match self.peeked.take() { + None => self.seq.next_element_seed(seed), + Some(Some(elem)) => { + let deserializer = private_de::ContentDeserializer::new(elem); + seed.deserialize(deserializer).map(Some) + } + Some(None) => Ok(None), + } + } + + fn size_hint(&self) -> Option { + self.seq.size_hint() + } +} + +impl<'de, S> From for PeekableSeqAccess<'de, S> +where + S: de::SeqAccess<'de>, +{ + fn from(seq: S) -> Self { + Self { seq, peeked: None } + } +} + +#[cfg(test)] +mod tests { + use std::fmt; + + use serde::de::{MapAccess, Visitor}; + use serde::{Deserialize, Deserializer}; + + #[test] + fn test_peek_and_consume() { + struct Test; + + impl<'de> Deserialize<'de> for Test { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TestVisitor; + + impl<'de> Visitor<'de> for TestVisitor { + type Value = Test; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map") + } + + fn visit_map(self, map: A) -> Result + where + A: MapAccess<'de>, + { + let mut map = super::PeekableMapAccess::from(map); + + // 1. Peek at the first key + let key: Option = map.peek_key()?; + assert_eq!(key.as_deref(), Some("a")); + + // 2. Consume the first key-value pair + let (key, value): (String, i32) = map.next_entry()?.unwrap(); + assert_eq!(key, "a"); + assert_eq!(value, 1); + + // 3. Peek at the second key + let key: Option = map.peek_key()?; + assert_eq!(key.as_deref(), Some("b")); + + // 4. Peek at the second value + let value: i32 = map.peek_value()?; + assert_eq!(value, 2); + + // 5. Peek at the second key again + let key: Option = map.peek_key()?; + assert_eq!(key.as_deref(), Some("b")); + + // 6. Consume the second key-value pair + let (key, value): (String, i32) = map.next_entry()?.unwrap(); + assert_eq!(key, "b"); + assert_eq!(value, 2); + + // 7. Ensure map is empty + assert!(map.next_key::()?.is_none()); + + Ok(Test) + } + } + + deserializer.deserialize_map(TestVisitor) + } + } + + let json = r#"{"a": 1, "b": 2}"#; + let _: Test = serde_json::from_str(json).unwrap(); + } + + #[test] + fn test_peek_value_first() { + struct Test; + + impl<'de> Deserialize<'de> for Test { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct TestVisitor; + + impl<'de> Visitor<'de> for TestVisitor { + type Value = Test; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("a map") + } + + fn visit_map(self, map: A) -> Result + where + A: MapAccess<'de>, + { + let mut map = super::PeekableMapAccess::from(map); + + // 1. Consume the first key + let key: String = map.next_key()?.unwrap(); + assert_eq!(key, "a"); + + // 2. Peek at the first value + let value: i32 = map.peek_value()?; + assert_eq!(value, 1); + + // 3. Peek at the first value again + let value: i32 = map.peek_value()?; + assert_eq!(value, 1); + + // 4. Consume the first value + let value: i32 = map.next_value()?; + assert_eq!(value, 1); + + // 5. Peek at the second key + let key: Option = map.peek_key()?; + assert_eq!(key.as_deref(), Some("b")); + + // 6. Consume the second key-value pair + let (key, value): (String, i32) = map.next_entry()?.unwrap(); + assert_eq!(key, "b"); + assert_eq!(value, 2); + + Ok(Test) + } + } + + deserializer.deserialize_map(TestVisitor) + } + } + + let json = r#"{"a": 1, "b": 2}"#; + let _: Test = serde_json::from_str(json).unwrap(); + } +} diff --git a/crates/spfs-cli/cmd-clean/Cargo.toml b/crates/spfs-cli/cmd-clean/Cargo.toml index 62a1c0c67d..d83a0627c9 100644 --- a/crates/spfs-cli/cmd-clean/Cargo.toml +++ b/crates/spfs-cli/cmd-clean/Cargo.toml @@ -29,4 +29,4 @@ spfs = { workspace = true } spfs-cli-common = { workspace = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread"] } tracing = { workspace = true } -url = { version = "2.2", features = ["serde"] } +url = { workspace = true, features = ["serde"] } diff --git a/crates/spk-build/src/archive_test.rs b/crates/spk-build/src/archive_test.rs index 134f0cf9f4..a4cd61c9ca 100644 --- a/crates/spk-build/src/archive_test.rs +++ b/crates/spk-build/src/archive_test.rs @@ -40,7 +40,9 @@ async fn test_archive_create_parents(#[case] solver: SolverImpl) { let filename = rt.tmpdir.path().join("deep/nested/path/archive.spk"); let repo = match &*rt.tmprepo { spk_solve::RepositoryHandle::SPFS(repo) => repo, - spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => { + spk_solve::RepositoryHandle::Mem(_) + | spk_solve::RepositoryHandle::Runtime(_) + | spk_solve::RepositoryHandle::Workspace(_) => { panic!("only spfs repositories are supported") } }; diff --git a/crates/spk-build/src/build/sources_test.rs b/crates/spk-build/src/build/sources_test.rs index 2b5510a847..f35d6d630f 100644 --- a/crates/spk-build/src/build/sources_test.rs +++ b/crates/spk-build/src/build/sources_test.rs @@ -75,6 +75,8 @@ async fn test_sources_subdir(tmpdir: tempfile::TempDir) { "user.name=Test User", "-c", "user.email=", + "-c", + "commit.gpgsign=false", "commit", "--author", "Test User ", diff --git a/crates/spk-cli/cmd-env/src/cmd_env.rs b/crates/spk-cli/cmd-env/src/cmd_env.rs index 83b0f25750..bcde9b7ea2 100644 --- a/crates/spk-cli/cmd-env/src/cmd_env.rs +++ b/crates/spk-cli/cmd-env/src/cmd_env.rs @@ -90,7 +90,7 @@ impl Run for Env { .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; - let solution = build_required_packages(&solution, solver).await?; + let solution = build_required_packages(&solution, &formatter, solver).await?; rt.status.editable = self.runtime.editable() || self.requests.any_build_stage_requests(&self.requested)?; diff --git a/crates/spk-cli/cmd-install/src/cmd_install.rs b/crates/spk-cli/cmd-install/src/cmd_install.rs index b3a53ad49a..3c93fe7179 100644 --- a/crates/spk-cli/cmd-install/src/cmd_install.rs +++ b/crates/spk-cli/cmd-install/src/cmd_install.rs @@ -119,7 +119,7 @@ impl Run for Install { } } - let compiled_solution = build_required_packages(&solution, solver) + let compiled_solution = build_required_packages(&solution, &formatter, solver) .await .wrap_err("Failed to build one or more packages from source")?; setup_current_runtime(&compiled_solution).await?; diff --git a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs index b7408fa80d..1fef9a3ecb 100644 --- a/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs +++ b/crates/spk-cli/cmd-make-recipe/src/cmd_make_recipe.rs @@ -3,9 +3,10 @@ // https://github.com/spkenv/spk use clap::Args; -use miette::{Context, IntoDiagnostic, Result}; +use miette::{Context, Result}; use spk_cli_common::{CommandArgs, Run, flags}; use spk_schema::foundation::format::FormatOptionMap; +use spk_schema::template::TemplateRenderConfig; use spk_schema::{SpecFileData, Template}; /// Render a package spec template into a recipe @@ -47,30 +48,29 @@ impl Run for MakeRecipe { let options = self.options.get_options()?; let mut workspace = self.workspace.load_or_default()?; - let configured = match self.package.as_ref() { + let template = match self.package.as_ref() { Some(p) => workspace.find_or_load_package_template(p), None => workspace.default_package_template().map_err(From::from), } .wrap_err("did not find recipe template")?; - if let Some(name) = configured.template.name() { + if let Some(name) = template.name() { tracing::info!("rendering template for {name}"); } else { tracing::info!("rendering template without a name"); } tracing::info!("using options {}", options.format_option_map()); - let data = spk_schema::TemplateData::new(&options); - tracing::debug!("full template data: {data:#?}"); - let rendered = spk_schema_tera::render_template( - configured.template.file_path().to_string_lossy(), - configured.template.source(), - &data, - ) - .into_diagnostic() - .wrap_err("Failed to render template")?; + let rendered = template + .render_to_string(TemplateRenderConfig { + options: options.clone(), + ..Default::default() + }) + .wrap_err("Failed to render template")?; print!("{rendered}"); - - match configured.template.render(&options) { + match template.render(TemplateRenderConfig { + options: options.clone(), + ..Default::default() + }) { Err(err) => { tracing::error!("This template did not render into a valid spec {err}"); Ok(1) diff --git a/crates/spk-cli/cmd-render/src/cmd_render.rs b/crates/spk-cli/cmd-render/src/cmd_render.rs index d1c6fde297..daf181f77d 100644 --- a/crates/spk-cli/cmd-render/src/cmd_render.rs +++ b/crates/spk-cli/cmd-render/src/cmd_render.rs @@ -55,7 +55,7 @@ impl Run for Render { .get_formatter(self.verbose)?; let solution = solver.run_and_print_resolve(&formatter).await?; - let solution = build_required_packages(&solution, solver.clone()).await?; + let solution = build_required_packages(&solution, &formatter, solver.clone()).await?; let stack = resolve_runtime_layers(true, &solution).await?; std::fs::create_dir_all(&self.target) .into_diagnostic() diff --git a/crates/spk-cli/cmd-workspace/Cargo.toml b/crates/spk-cli/cmd-workspace/Cargo.toml new file mode 100644 index 0000000000..ec40005f1b --- /dev/null +++ b/crates/spk-cli/cmd-workspace/Cargo.toml @@ -0,0 +1,37 @@ +[package] +authors = { workspace = true } +edition = { workspace = true } +name = "spk-cmd-workspace" +version = { workspace = true } +license-file = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +readme = { workspace = true } +description = { workspace = true } + +[lints] +workspace = true + +[features] +migration-to-components = [ + "spk-cli-common/migration-to-components", + "spk-cmd-make-binary/migration-to-components", + "spk-cmd-make-source/migration-to-components", +] + +[dependencies] +async-trait = { workspace = true } +clap = { workspace = true } +itertools = { workspace = true } +miette = { workspace = true, features = ["fancy"] } +spfs = { workspace = true } +spk-build = { workspace = true } +spk-cli-common = { workspace = true } +spk-cmd-make-binary = { workspace = true } +spk-cmd-make-source = { workspace = true } +spk-workspace = { workspace = true } +spk-schema = { workspace = true } +spk-solve = { workspace = true } +spk-storage = { workspace = true } +tracing = { workspace = true } +colored = { workspace = true } diff --git a/crates/spk-cli/cmd-workspace/src/build.rs b/crates/spk-cli/cmd-workspace/src/build.rs new file mode 100644 index 0000000000..d9b1bb407b --- /dev/null +++ b/crates/spk-cli/cmd-workspace/src/build.rs @@ -0,0 +1,200 @@ +use std::sync::Arc; +use std::vec; + +use clap::Args; +use miette::{Context, IntoDiagnostic, Result}; +use spk_cli_common::{CommandArgs, Run, build_required_packages, flags}; +use spk_schema::foundation::format::FormatIdent; +use spk_schema::ident::{RangeIdent, RequestWithOptions, ToAnyIdentWithoutBuild, VarRequest}; +use spk_schema::name::RepositoryName; +use spk_schema::v1::{Override, PlatformRequirement}; +use spk_schema::{ApiVersion, SpecFileData, SpecRecipe, Template, VersionIdent}; +use spk_solve::{Package, PkgRequest, SolverExt, SolverMut}; + +/// Build a set of packages from this workspace +#[derive(Args, Clone)] +#[clap(visible_aliases = &["b"])] +pub struct Build { + #[clap(flatten)] + runtime: flags::Runtime, + #[clap(flatten)] + workspace: flags::Workspace, + #[clap(flatten)] + solver: flags::Solver, + #[clap(flatten)] + options: flags::Options, + + #[clap(short, long, global = true, action = clap::ArgAction::Count)] + pub verbose: u8, + + /// The platform to build, which defines the set of packages + platform: String, +} + +#[async_trait::async_trait] +impl Run for Build { + type Output = i32; + + async fn run(&mut self) -> Result { + self.runtime.ensure_active_runtime(&["b", "build"]).await?; + + let mut workspace = self.workspace.load_or_default()?; + let options = self.options.get_options()?; + let template = workspace.find_or_load_package_template(&self.platform)?; + let recipe = template + .render(spk_schema::template::TemplateRenderConfig { + options, + ..Default::default() + })? + .into_recipe()?; + let SpecRecipe::V1Platform(platform) = &*recipe else { + miette::bail!( + "Only {} recipe files can be used to build workspaces", + ApiVersion::V1Platform + ) + }; + // currently, no other solver is able to properly handle + // source builds, which we need in order to determine what + // to build from the workspace. + self.solver.decision_formatter_settings.solver_to_run = flags::SolverToRun::Cli; + let mut solver = self.solver.get_solver(&self.options).await?; + + let root = match workspace.root() { + Some(root) => root.to_owned(), + None => std::env::current_dir().into_diagnostic()?, + }; + let repo_name = RepositoryName::new("workspace")?; + + tracing::info!("Adding requests for all platform requirements:"); + let requested_by = spk_solve::RequestedBy::BinaryBuild( + platform.platform.to_build_ident(spk_solve::Build::Source), + ); + for requirement in platform.requirements.iter() { + let request = match requirement { + PlatformRequirement::Pkg(pkg) => { + let Some(build) = pkg.build.as_ref() else { + continue; + }; + let templates = workspace.find_package_templates_mut(&pkg.pkg); + if templates.is_empty() { + miette::bail!( + "Cannot build '{}', no spec files found in workspace", + pkg.pkg + ); + } + + let to_build = VersionIdent::new(pkg.pkg.clone(), build.version.clone()) + .to_any_ident_without_build(); + let range_ident = RangeIdent::equals(&to_build, None) + .with_repository(Some(repo_name.to_owned())); + tracing::info!(" > pkg: {range_ident}"); + RequestWithOptions::from(PkgRequest::new(range_ident, requested_by.clone())) + } + PlatformRequirement::Var(var) => { + let Some(Override::Replace(value)) = &var.at_build else { + continue; + }; + let inner = VarRequest { + var: var.var.clone(), + value: Arc::from(value.as_str()), + description: None, + }; + tracing::info!(" > var: {inner}"); + RequestWithOptions::from(inner) + } + }; + solver.add_request(request); + } + + let local = Arc::::new( + spk_storage::local_repository().await?.into(), + ); + let workspace_repo_handle = Arc::new(spk_storage::RepositoryHandle::Workspace( + spk_storage::WorkspaceRepository::new(&root, repo_name.to_owned(), workspace), + )); + // we still need a reference to the underlying workspace instance for later + let spk_storage::RepositoryHandle::Workspace(workspace_repo) = &*workspace_repo_handle + else { + unreachable!() + }; + solver.add_repository(Arc::clone(&workspace_repo_handle)); + let formatter = self + .solver + .decision_formatter_settings + .get_formatter(self.verbose)?; + // the key to this whole process is that we allow the solver to + // imagine the builds that this workspace can produce + solver.set_binary_only(false); + let solution = solver.run_and_print_resolve(&formatter).await?; + + for solved in solution.items() { + if !solved.is_source_build() + || solved + .repo_name() + .as_deref() + .is_some_and(|n| n != repo_name) + { + continue; + }; + // TODO: be more intelligent about when this is needed? + // ideally we can have SOME sense of if the recipe + // has changed on disk, but there's complexity + // around checking the source files themselves + let ident = solved.spec.ident(); + tracing::info!( + "Generating source package for required build: {}", + ident.format_ident() + ); + + let templates = workspace_repo.find_package_template_for_version( + ident.name(), + spk_schema::version_range::DoubleEqualsVersion::new(ident.version().clone()), + ); + if templates.len() != 1 { + miette::bail!( + "Expected exactly one package template for {}, found {}", + ident.format_ident(), + templates.len() + ); + } + let template = templates[0]; + let recipe = template.render(spk_schema::template::TemplateRenderConfig { + version: Some(ident.version().clone()), + ..Default::default() + })?; + let SpecFileData::Recipe(recipe) = recipe else { + miette::bail!( + "Expected recipe from file {}", + template.file_path().display() + ) + }; + + tracing::info!("saving package recipe for {}", ident.format_ident()); + local.force_publish_recipe(&recipe).await?; + + tracing::info!("collecting sources for {}", ident.format_ident()); + let (out, _components) = + spk_build::SourcePackageBuilder::from_recipe(Arc::unwrap_or_clone(recipe)) + .build_and_publish( + &template + .file_path() + .parent() + .unwrap_or_else(|| template.file_path()), + &*local, + ) + .await + .wrap_err("Failed to collect sources")?; + tracing::info!("created {}", out.ident().format_ident()); + } + + build_required_packages(&solution, &formatter, solver).await?; + + Ok(0) + } +} + +impl CommandArgs for Build { + fn get_positional_args(&self) -> Vec { + vec![self.platform.clone()] + } +} diff --git a/crates/spk-cli/cmd-workspace/src/cmd_workspace.rs b/crates/spk-cli/cmd-workspace/src/cmd_workspace.rs new file mode 100644 index 0000000000..a014262fb3 --- /dev/null +++ b/crates/spk-cli/cmd-workspace/src/cmd_workspace.rs @@ -0,0 +1,43 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use clap::{Args, Subcommand}; +use miette::Result; +use spk_cli_common::{CommandArgs, Run}; + +/// Query and operate on an spk workspace directory. +#[derive(Args, Clone)] +#[clap(visible_aliases = &["ws", "w"])] +pub struct Workspace { + #[clap(subcommand)] + pub cmd: Command, +} + +#[derive(Subcommand, Clone)] +#[allow(clippy::large_enum_variant)] +pub enum Command { + Info(crate::info::Info), + Build(crate::build::Build), +} + +#[async_trait::async_trait] +impl Run for Workspace { + type Output = i32; + + async fn run(&mut self) -> Result { + match &mut self.cmd { + Command::Info(cmd) => cmd.run().await, + Command::Build(cmd) => cmd.run().await, + } + } +} + +impl CommandArgs for Workspace { + fn get_positional_args(&self) -> Vec { + match &self.cmd { + Command::Info(cmd) => cmd.get_positional_args(), + Command::Build(cmd) => cmd.get_positional_args(), + } + } +} diff --git a/crates/spk-cli/cmd-workspace/src/info.rs b/crates/spk-cli/cmd-workspace/src/info.rs new file mode 100644 index 0000000000..feea6599a4 --- /dev/null +++ b/crates/spk-cli/cmd-workspace/src/info.rs @@ -0,0 +1,73 @@ +use clap::Args; +use colored::Colorize; +use itertools::Itertools; +use miette::{IntoDiagnostic, Result}; +use spk_cli_common::{CommandArgs, Run}; +use spk_schema::Template; +use spk_schema::template::DiscoverVersions; + +/// Print information about the current workspace +#[derive(Args, Clone)] +#[clap(visible_aliases = &["i"])] +pub struct Info {} + +#[async_trait::async_trait] +impl Run for Info { + type Output = i32; + + async fn run(&mut self) -> Result { + let mut root = std::env::current_dir().into_diagnostic()?; + let workspace = match spk_workspace::WorkspaceFile::discover(".") { + Ok((file, path)) => { + root = path; + spk_workspace::Workspace::builder() + .load_from_file(file)? + .build()? + } + Err(spk_workspace::error::LoadWorkspaceFileError::WorkspaceNotFound(_)) => { + tracing::warn!( + "Workspace file not found using the current path, loading ephemerally" + ); + spk_workspace::Workspace::builder() + .load_from_current_dir()? + .build()? + } + Err(err) => return Err(err.into()), + }; + println!("root: {}", root.display()); + println!("packages:"); + // we'd like the items in the workspace to be sorted alphabetically + let mut packages = workspace.iter().collect_vec(); + packages.sort_by(|a, b| a.0.cmp(b.0)); + for (pkg, tpl) in packages { + let mut versions = tpl + .discover_versions()? + .iter() + .map(|v| v.to_string()) + .collect_vec(); + if versions.len() > 5 { + let tail = format!("and {} more...", versions.len() - 5); + versions.truncate(5); + versions.push(tail); + } + let path = tpl.file_path(); + // try to get a relative workspace path, if possible + let path = path + .strip_prefix(&root) + .unwrap_or(path) + .display() + .to_string(); + println!(". {} {}", pkg.bold(), path.dimmed()); + if !versions.is_empty() { + println!(" {}", versions.join(", ")); + }; + } + Ok(0) + } +} + +impl CommandArgs for Info { + fn get_positional_args(&self) -> Vec { + Vec::default() + } +} diff --git a/crates/spk-cli/cmd-workspace/src/lib.rs b/crates/spk-cli/cmd-workspace/src/lib.rs new file mode 100644 index 0000000000..fbc66202e7 --- /dev/null +++ b/crates/spk-cli/cmd-workspace/src/lib.rs @@ -0,0 +1,8 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +mod build; +mod info; + +pub mod cmd_workspace; diff --git a/crates/spk-cli/common/src/exec.rs b/crates/spk-cli/common/src/exec.rs index f753b6f7a0..e6d3a9e207 100644 --- a/crates/spk-cli/common/src/exec.rs +++ b/crates/spk-cli/common/src/exec.rs @@ -8,7 +8,7 @@ use spk_build::BinaryPackageBuilder; use spk_schema::Package; use spk_schema::foundation::format::{FormatIdent, FormatOptionMap}; use spk_solve::solution::{PackageSource, Solution}; -use spk_solve::{SolverExt, SolverMut}; +use spk_solve::{DecisionFormatter, SolverExt, SolverMut}; use spk_storage as storage; use crate::Result; @@ -18,6 +18,7 @@ use crate::Result; /// Returns a new solution of only binary packages. pub async fn build_required_packages( solution: &Solution, + formatter: &DecisionFormatter, solver: Solver, ) -> Result where @@ -29,8 +30,8 @@ where let options = solution.options(); let mut compiled_solution = Solution::new(options.clone()); for item in solution.items() { - let recipe = match &item.source { - PackageSource::BuildFromSource { recipe } => recipe, + let (recipe, repo) = match &item.source { + PackageSource::BuildFromSource { recipe, repo } => (recipe, repo), source => { compiled_solution.add(item.request.clone(), Arc::clone(&item.spec), source.clone()); continue; @@ -44,11 +45,14 @@ where ); let (package, components) = BinaryPackageBuilder::from_recipe_with_solver((**recipe).clone(), solver.clone()) + .with_repository(Arc::clone(repo)) .with_repositories(repos.clone()) + .with_source_formatter(formatter.clone()) + .with_build_formatter(formatter.clone()) .build_and_publish(&options, &*local_repo) .await?; let source = PackageSource::Repository { - repo: local_repo.clone(), + repo: Arc::clone(&local_repo), components, }; compiled_solution.add(item.request.clone(), Arc::new(package), source); diff --git a/crates/spk-cli/common/src/flags.rs b/crates/spk-cli/common/src/flags.rs index 37cf3d8abf..5de9b8b698 100644 --- a/crates/spk-cli/common/src/flags.rs +++ b/crates/spk-cli/common/src/flags.rs @@ -430,14 +430,18 @@ impl Requests { unreachable!(); }; - let configured = ws + let template = ws .find_or_load_package_template(package) .wrap_err("did not find recipe template")?; - let rendered_data = configured.template.render(options)?; + let rendered_data = + template.render(spk_schema::template::TemplateRenderConfig { + options: options.clone(), + ..Default::default() + })?; let recipe = rendered_data.into_recipe().wrap_err_with(|| { format!( "{filename} was expected to contain a recipe", - filename = configured.template.file_path().to_string_lossy() + filename = template.file_path().to_string_lossy() ) })?; idents.push(recipe.ident().to_any_ident(None)); @@ -763,7 +767,8 @@ impl Workspace { | Err(spk_workspace::error::FromPathError::LoadWorkspaceFileError( spk_workspace::error::LoadWorkspaceFileError::WorkspaceNotFound(_), )) => { - let mut builder = spk_workspace::Workspace::builder(); + let mut builder = + spk_workspace::Workspace::builder().with_ignore_invalid_files(true); if self.workspace.is_dir() { tracing::debug!( @@ -911,7 +916,7 @@ where Some(package_name) => workspace.find_or_load_package_template(package_name), None => workspace.default_package_template().map_err(From::from), }; - let configured = match from_workspace { + let template = match from_workspace { Ok(template) => template, res @ Err(FindOrLoadPackageTemplateError::FindPackageTemplateError( FindPackageTemplateError::MultipleTemplates(_), @@ -983,17 +988,22 @@ where } } }; - let found = configured.template.render(options).wrap_err_with(|| { - format!( - "{filename} was expected to contain a valid spk yaml data file", - filename = configured.template.file_path().to_string_lossy() - ) - })?; + let found = template + .render(spk_schema::template::TemplateRenderConfig { + options: options.clone(), + ..Default::default() + }) + .wrap_err_with(|| { + format!( + "{filename} was expected to contain a valid spk yaml data file", + filename = template.file_path().to_string_lossy() + ) + })?; tracing::debug!( - "Rendered configured.template from the data in {:?}", - configured.template.file_path() + "Rendered template from the data in {:?}", + template.file_path() ); - Ok((found, configured.template.file_path().to_owned())) + Ok((found, template.file_path().to_owned())) } #[derive(Args, Clone)] diff --git a/crates/spk-cli/group3/src/cmd_export.rs b/crates/spk-cli/group3/src/cmd_export.rs index 02147bf4da..a24e31178d 100644 --- a/crates/spk-cli/group3/src/cmd_export.rs +++ b/crates/spk-cli/group3/src/cmd_export.rs @@ -52,7 +52,9 @@ impl Run for Export { .iter() .map(|repo| match &**repo { storage::RepositoryHandle::SPFS(repo) => Ok(repo), - storage::RepositoryHandle::Mem(_) | storage::RepositoryHandle::Runtime(_) => { + storage::RepositoryHandle::Mem(_) + | storage::RepositoryHandle::Runtime(_) + | storage::RepositoryHandle::Workspace(_) => { bail!("Only spfs repositories are supported") } }) diff --git a/crates/spk-cli/group3/src/cmd_import_test.rs b/crates/spk-cli/group3/src/cmd_import_test.rs index 613c99fca1..a4d3135543 100644 --- a/crates/spk-cli/group3/src/cmd_import_test.rs +++ b/crates/spk-cli/group3/src/cmd_import_test.rs @@ -42,7 +42,9 @@ async fn test_archive_io(#[case] solver: SolverImpl) { filename.ensure(); let repo = match &*rt.tmprepo { spk_solve::RepositoryHandle::SPFS(repo) => repo, - spk_solve::RepositoryHandle::Mem(_) | spk_solve::RepositoryHandle::Runtime(_) => { + spk_solve::RepositoryHandle::Mem(_) + | spk_solve::RepositoryHandle::Runtime(_) + | spk_solve::RepositoryHandle::Workspace(_) => { panic!("only spfs repositories are supported") } }; diff --git a/crates/spk-cli/group4/src/cmd_lint.rs b/crates/spk-cli/group4/src/cmd_lint.rs index f4c8936ba9..106c669aa1 100644 --- a/crates/spk-cli/group4/src/cmd_lint.rs +++ b/crates/spk-cli/group4/src/cmd_lint.rs @@ -28,7 +28,12 @@ impl Run for Lint { let options = self.options.get_options()?; let mut out = 0; for spec in self.packages.iter() { - let result = SpecTemplate::from_file(spec).and_then(|t| t.render(&options)); + let result = SpecTemplate::from_file(spec).and_then(|t| { + t.render(spk_schema::template::TemplateRenderConfig { + options: options.clone(), + ..Default::default() + }) + }); match result { Ok(_) => println!("{} {}", "OK".green(), spec.display()), Err(err) => { diff --git a/crates/spk-cli/group4/src/cmd_view.rs b/crates/spk-cli/group4/src/cmd_view.rs index 2a45acb419..8cf87c9237 100644 --- a/crates/spk-cli/group4/src/cmd_view.rs +++ b/crates/spk-cli/group4/src/cmd_view.rs @@ -495,16 +495,19 @@ impl View { workspace: &mut spk_workspace::Workspace, show_variants_with_tests: bool, ) -> Result { - let configured = match self.package.as_ref() { + let template = match self.package.as_ref() { Some(name) => workspace.find_or_load_package_template(name), None => workspace.default_package_template().map_err(From::from), } .wrap_err("did not find recipe template")?; - let rendered_data = configured.template.render(options)?; + let rendered_data = template.render(spk_schema::template::TemplateRenderConfig { + options: options.clone(), + ..Default::default() + })?; let recipe = rendered_data.into_recipe().wrap_err_with(|| { format!( "{filename} was expected to contain a recipe", - filename = configured.template.file_path().to_string_lossy() + filename = template.file_path().to_string_lossy() ) })?; diff --git a/crates/spk-schema/Cargo.toml b/crates/spk-schema/Cargo.toml index 2e4e184aa5..6d1e1c97df 100644 --- a/crates/spk-schema/Cargo.toml +++ b/crates/spk-schema/Cargo.toml @@ -16,6 +16,7 @@ workspace = true migration-to-components = ["spk-schema-foundation/migration-to-components"] [dependencies] +bracoxide = { workspace = true } config = { workspace = true } data-encoding = "2.3" dunce = { workspace = true } @@ -31,11 +32,13 @@ itertools = { workspace = true } miette = { workspace = true } nom = { workspace = true } nom-supreme = { workspace = true } +once_cell = { workspace = true } regex = { workspace = true } relative-path = { workspace = true } ring = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +serde-peekable = { workspace = true } serde_yaml = { workspace = true } shellexpand = "3.1.0" spfs = { workspace = true } @@ -48,6 +51,7 @@ tempfile = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } variantly = { workspace = true } +url = { workspace = true, features = ["serde"] } [dev-dependencies] proptest = "1.0.0" diff --git a/crates/spk-schema/crates/foundation/src/ident/range_ident.rs b/crates/spk-schema/crates/foundation/src/ident/range_ident.rs index 63b77f0640..bcdefb12f2 100644 --- a/crates/spk-schema/crates/foundation/src/ident/range_ident.rs +++ b/crates/spk-schema/crates/foundation/src/ident/range_ident.rs @@ -89,6 +89,12 @@ impl RangeIdent { } } + /// Set the repository name for this range ident. + pub fn with_repository(mut self, repository: Option) -> Self { + self.repository_name = repository; + self + } + /// Create a range ident that requests the identified package using `==` semantics. /// /// The returned range will request the identified components of the given package. diff --git a/crates/spk-schema/crates/foundation/src/option_map/mod.rs b/crates/spk-schema/crates/foundation/src/option_map/mod.rs index c917884352..ca1a0ad591 100644 --- a/crates/spk-schema/crates/foundation/src/option_map/mod.rs +++ b/crates/spk-schema/crates/foundation/src/option_map/mod.rs @@ -260,27 +260,32 @@ impl OptionMap { /// mapping instead. In the case where there is a value for both `python` and `python.abi` /// the former will be dropped. pub fn to_yaml_value_expanded(&self) -> serde_yaml::Mapping { + self.clone().into_yaml_value_expanded() + } + + /// Like [`OptionMap::to_yaml_value_expanded`], but consumes the options to save clone operations. + pub fn into_yaml_value_expanded(self) -> serde_yaml::Mapping { use serde_yaml::{Mapping, Value}; let mut yaml = Mapping::default(); - for (key, value) in self.iter() { - let target = match key.namespace() { - Some(ns) => { - let ns = Value::String(ns.to_string()); - let ns_value = yaml - .entry(ns) - .or_insert_with(|| serde_yaml::Value::Mapping(Default::default())); - if ns_value.as_mapping().is_none() { - *ns_value = serde_yaml::Value::Mapping(Default::default()); - } - ns_value - .as_mapping_mut() - .expect("already validated that this is a mapping") - } - None => &mut yaml, + for (key, value) in self.options.into_iter() { + let value = serde_yaml::Value::String(value); + let Some(ns) = key.namespace() else { + // when there is no namespace, efficiently reuse the existing strings + yaml.insert(key.into_inner().into(), value); + continue; }; + let ns = Value::String(ns.to_string()); + let ns_value = yaml + .entry(ns) + .or_insert_with(|| serde_yaml::Value::Mapping(Default::default())); + if ns_value.as_mapping().is_none() { + *ns_value = serde_yaml::Value::Mapping(Default::default()); + } + let ns_map = ns_value + .as_mapping_mut() + .expect("already validated that this is a mapping"); let key = serde_yaml::Value::String(key.base_name().to_string()); - let value = serde_yaml::Value::String(value.to_string()); - target.insert(key, value); + ns_map.insert(key, value); } yaml } diff --git a/crates/spk-schema/crates/foundation/src/version/compat/problems.rs b/crates/spk-schema/crates/foundation/src/version/compat/problems.rs index 750b5567bb..5f5eecf2fc 100644 --- a/crates/spk-schema/crates/foundation/src/version/compat/problems.rs +++ b/crates/spk-schema/crates/foundation/src/version/compat/problems.rs @@ -163,8 +163,6 @@ pub enum PackageRepoProblem { to_string = "package did not come from requested repo (it was embedded in {parent_ident})" )] EmbeddedInPackageFromWrongRepository { parent_ident: String }, - #[strum(to_string = "package did not come from requested repo (it comes from a spec)")] - FromRecipeFromWrongRepository, #[strum( to_string = "package did not come from requested repo (it comes from an internal test setup)" )] diff --git a/crates/spk-schema/crates/foundation/src/version/mod.rs b/crates/spk-schema/crates/foundation/src/version/mod.rs index 221102fe24..acc9b9f605 100644 --- a/crates/spk-schema/crates/foundation/src/version/mod.rs +++ b/crates/spk-schema/crates/foundation/src/version/mod.rs @@ -64,6 +64,15 @@ pub const TAG_SEP: &str = "."; pub const SENTINEL_LABEL: &str = "Tail"; pub const POSITION_LABELS: &[&str] = &["Major", "Minor", "Patch"]; +/// A macro to create a new version from a string literal, panics if the version is invalid. +#[macro_export] +macro_rules! version { + ($version:literal) => { + <$crate::version::Version as ::std::str::FromStr>::from_str($version) + .expect("invalid version") + }; +} + /// Returns the name of the version component at the given position. /// /// Position zero corresponds to 'Major', 1 to 'Minor' and so on. diff --git a/crates/spk-schema/crates/tera/Cargo.toml b/crates/spk-schema/crates/tera/Cargo.toml index 511ba05a00..448f762992 100644 --- a/crates/spk-schema/crates/tera/Cargo.toml +++ b/crates/spk-schema/crates/tera/Cargo.toml @@ -18,13 +18,13 @@ migration-to-components = ["spk-schema-foundation/migration-to-components"] [dependencies] miette = { workspace = true } once_cell = { workspace = true } -regex = "1.6.0" -serde = "1.0" -serde_json = "1.0" +regex = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } spk-schema-foundation = { workspace = true } tera = "1.19.1" thiserror = { workspace = true } -tracing = "0.1.35" +tracing = { workspace = true } [dev-dependencies] rstest = { workspace = true } diff --git a/crates/spk-schema/src/error.rs b/crates/spk-schema/src/error.rs index b9b6d57e95..5d34fb805e 100644 --- a/crates/spk-schema/src/error.rs +++ b/crates/spk-schema/src/error.rs @@ -71,6 +71,14 @@ pub enum Error { #[error(transparent)] #[diagnostic(forward(0))] SpkConfigError(#[from] spk_config::Error), + + #[error("Failed to run git command: {0}")] + GitCommandFailed(#[from] std::io::Error), + #[error("git command failed with exit code {0}: {1}")] + GitCommandExited(i32, String), + #[error("Failed to parse discovered tag as version: {0}")] + #[diagnostic(help = "consider adding or adjusting the tag regex and extraction pattern(s)")] + FailedToParseTagAsVersion(spk_schema_foundation::version::Error), } impl Error { diff --git a/crates/spk-schema/src/lib.rs b/crates/spk-schema/src/lib.rs index cda47205a9..45ada22235 100644 --- a/crates/spk-schema/src/lib.rs +++ b/crates/spk-schema/src/lib.rs @@ -20,7 +20,7 @@ mod recipe; mod requirements_list; mod source_spec; mod spec; -mod template; +pub mod template; mod test; pub mod v0; pub mod v1; @@ -77,7 +77,7 @@ pub use spk_schema_foundation::{ version, version_range, }; -pub use template::{Template, TemplateData, TemplateExt}; +pub use template::{Template, TemplateExt}; pub use test::{Test, TestStage}; pub use v0::{AutoHostVars, RecipeComponentSpec, Script}; pub use validation::{ValidationRule, ValidationSpec}; diff --git a/crates/spk-schema/src/requirements_list.rs b/crates/spk-schema/src/requirements_list.rs index e0ee9eafe3..aac8273f11 100644 --- a/crates/spk-schema/src/requirements_list.rs +++ b/crates/spk-schema/src/requirements_list.rs @@ -528,7 +528,7 @@ where impl std::fmt::Display for OriginalError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "failed to deserialize request") + write!(f, "failed to deserialize request: {}", self.source) } } diff --git a/crates/spk-schema/src/source_spec_test.rs b/crates/spk-schema/src/source_spec_test.rs index 3fd91855ad..e9634fbd25 100644 --- a/crates/spk-schema/src/source_spec_test.rs +++ b/crates/spk-schema/src/source_spec_test.rs @@ -73,6 +73,8 @@ fn test_git_sources(tmpdir: tempfile::TempDir) { "user.name=Test User", "-c", "user.email=", + "-c", + "commit.gpgsign=false", "commit", "--author", "Test User ", diff --git a/crates/spk-schema/src/spec.rs b/crates/spk-schema/src/spec.rs index 330b0e07ed..993b363801 100644 --- a/crates/spk-schema/src/spec.rs +++ b/crates/spk-schema/src/spec.rs @@ -3,7 +3,7 @@ // https://github.com/spkenv/spk use std::borrow::Cow; -use std::collections::HashSet; +use std::collections::BTreeSet; use std::io::Read; use std::path::Path; use std::str::FromStr; @@ -32,6 +32,7 @@ use crate::foundation::version::{Compat, Compatibility, Version}; use crate::ident::{PinnedRequest, Satisfy, VarRequest}; use crate::metadata::Meta; use crate::package::{DownstreamRequirements, OptionValues}; +use crate::template::{DiscoverVersions, TemplateRenderConfig, TemplateSpec}; use crate::{ BuildEnv, ComponentSpec, @@ -146,13 +147,23 @@ macro_rules! spec { }}; } -/// A generic, structured data object that can be turned into a recipe -/// when provided with the necessary option values +/// A recipe template that can be rendered into a full recipe spec. +/// +/// This struct represents a spec file that may contain a top-level `template` block, +/// which defines how to discover versions and render the recipe for a specific version. #[derive(Debug, Clone)] pub struct SpecTemplate { + /// The structured metadata from the `template:` block. + template_spec: TemplateSpec, + /// Cached set of discovered versions, to avoid costly re-discovery. + supported_versions: once_cell::sync::OnceCell>, + /// The name of the api item that this template is for, if any. + /// + /// Some api items are unnamed, in which case this will be `None`. name: Option, - versions: HashSet, + /// The path to the file this template was loaded from. file_path: std::path::PathBuf, + /// The full source of the file this template was loaded from. template: Arc, } @@ -166,20 +177,13 @@ impl SpecTemplate { pub fn name(&self) -> Option<&PkgNameBuf> { self.name.as_ref() } +} - /// The versions that are available to create with this template. - /// - /// An empty set does not signify no versions, but rather that - /// nothing has been specified or discerned. - pub fn versions(&self) -> &HashSet { - &self.versions - } - - /// Clear and reset the versions that are available to create - /// with this template. - pub fn set_versions(&mut self, versions: impl IntoIterator) { - self.versions.clear(); - self.versions.extend(versions); +impl DiscoverVersions for SpecTemplate { + fn discover_versions(&self) -> Result> { + self.supported_versions + .get_or_try_init(|| self.template_spec.versions.discover_versions()) + .cloned() } } @@ -188,8 +192,35 @@ impl Template for SpecTemplate { &self.file_path } - fn render(&self, options: &OptionMap) -> Result { - let data = super::TemplateData::new(options); + fn render_to_string(&self, data: TemplateRenderConfig) -> Result { + let TemplateRenderConfig { + version, + options, + environment, + } = data; + let supported_versions = self.discover_versions()?; + if supported_versions.is_empty() { + return Err(Error::String("No versions found".to_string())); + } + let version = match version { + Some(v) if supported_versions.contains(&v) => v, + Some(v) => { + return Err(Error::String(format!( + "Requested version '{v}' is not supported by this template" + ))); + } + None if supported_versions.len() == 1 => supported_versions + .into_iter() + .next() + .expect("len was validated"), + None => { + return Err(Error::String( + "No version specified, but multiple versions are supported".to_string(), + )); + } + }; + + let data = super::template::TemplateData::new(version, options, environment); let rendered = spk_schema_tera::render_template( self.file_path.to_string_lossy(), &self.template, @@ -197,8 +228,7 @@ impl Template for SpecTemplate { ) .map_err(Error::InvalidTemplate)?; - let file_data = SpecFileData::from_yaml(rendered)?; - Ok(file_data) + Ok(rendered) } } @@ -215,9 +245,17 @@ impl TemplateExt for SpecTemplate { .read_to_string(&mut template) .map_err(|err| Error::String(format!("Failed to read file {path:?}: {err}")))?; + #[derive(serde::Deserialize)] + struct PartialTemplate { + api: Option, + template: Option, + #[serde(flatten)] + body: serde_yaml::Mapping, + } + // validate that the template is still a valid yaml mapping even // though we will need to re-process it again later on - let template_value: serde_yaml::Mapping = match serde_yaml::from_str(&template) { + let partial = match serde_yaml::from_str::(&template) { Err(err) => { return Err(Error::InvalidYaml(SerdeError::new( template, @@ -227,9 +265,7 @@ impl TemplateExt for SpecTemplate { Ok(v) => v, }; - let api = template_value.get(serde_yaml::Value::String("api".to_string())); - - if api.is_none() { + let api = partial.api.unwrap_or_else(|| { tracing::warn!( spec_file = %file_path.to_string_lossy(), "Spec file is missing the 'api' field, this may be an error in the future" @@ -237,46 +273,59 @@ impl TemplateExt for SpecTemplate { tracing::warn!( " > for package specs in the original spk format, add a 'api: v0/package' line" ); - } + ApiVersion::V0Package + }); let name_field = match api { - Some(serde_yaml::Value::String(api)) => { - let field = api.split("/").nth(1).unwrap_or("pkg"); - if field == "package" { "pkg" } else { field } - } - Some(_) => "pkg", - None => "pkg", + ApiVersion::V0Package => Some("pkg"), + ApiVersion::V0Platform | ApiVersion::V1Platform => Some("platform"), + ApiVersion::V0Requirements => None, }; - let name = if name_field == "requirements" { - // This is Spec data and does not have a name, e.g. Requests(V0::Requirements) - None - } else { - // Read the name from the name field, check it is a valid - // string, and turn it into a PkgNameBuf - let pkg = template_value - .get(serde_yaml::Value::String(name_field.to_string())) - .ok_or_else(|| { + let (name, version) = match name_field { + None => (None, None), + Some(name_field) => { + // Read the name from the name field, check it is a valid + // string, and turn it into a PkgNameBuf + let name = partial.body.get(name_field).ok_or_else(|| { crate::Error::String(format!( "Missing '{name_field}' field in spec file: {file_path:?}" )) })?; - let pkg = pkg.as_str().ok_or_else(|| { - crate::Error::String(format!( - "Invalid value for '{name_field}' field: expected string, got {pkg:?} in {file_path:?}" - )) - })?; + let name = name.as_str().ok_or_else(|| { + crate::Error::String(format!( + "Invalid value for '{name_field}' field: expected string, got {name:?} in {file_path:?}" + )) + })?; + + let mut components = name.split('/'); + // it should never be possible for split to return 0 results + // but this trick avoids the use of unwrap + let name = components.next().unwrap_or(name); + let version = components.next().unwrap_or(""); + + ( + Some(PkgNameBuf::from_str(name)?), + Version::from_str(version).ok(), + ) + } + }; - // it should never be possible for split to return 0 results - // but this trick avoids the use of unwrap - Some(PkgNameBuf::from_str(pkg.split('/').next().unwrap_or(pkg))?) + let template_spec = match (partial.template, version) { + (Some(mut template_spec), v) => { + template_spec.versions.in_spec = v; + template_spec + } + (None, Some(v)) => TemplateSpec::from_single_version(v), + (None, None) => TemplateSpec::from_single_version(Version::default()), }; Ok(Self { + template_spec, + supported_versions: Default::default(), file_path, name, - versions: Default::default(), template: template.into(), }) } @@ -476,6 +525,11 @@ impl FromYaml for SpecRecipe { .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; Ok(Self::V0Platform(inner)) } + ApiVersion::V1Platform => { + let inner = serde_yaml::from_str(&yaml) + .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; + Ok(Self::V1Platform(inner)) + } ApiVersion::V0Requirements => { // Reading a list of requests/requirements file is not // supported here. But it might be in future. @@ -532,11 +586,12 @@ impl SpecFileData { } } - pub fn from_yaml>(yaml: S) -> Result { - let yaml = yaml.into(); + /// Parse the provided string as a yaml-encoded [`SpecFileData`]. + pub fn from_yaml>(yaml: S) -> Result { + let yaml = yaml.as_ref(); let value: serde_yaml::Value = - serde_yaml::from_str(&yaml).map_err(Error::SpecEncodingError)?; + serde_yaml::from_str(yaml).map_err(Error::SpecEncodingError)?; // First work out what kind of data this is, based on the // DataApiVersionMapping value. @@ -545,26 +600,33 @@ impl SpecFileData { // to understand that we only pass ownership of 'yaml' if // the function is returning Err(err) => { - return Err(SerdeError::new(yaml, SerdeYamlError(err)).into()); + return Err(SerdeError::new(yaml.to_owned(), SerdeYamlError(err)).into()); } Ok(m) => m, }; - // Create the appropriate object from the parsed value + // Create the appropriate object from the original yaml + // NOTE: parsing the yaml value as a string again is more work but + // provides contextualized error messages that highlight source locations let spec = match with_version.api { ApiVersion::V0Package => { - let inner = serde_yaml::from_value(value) - .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; + let inner = serde_yaml::from_str(yaml) + .map_err(|err| SerdeError::new(yaml.to_owned(), SerdeYamlError(err)))?; SpecFileData::Recipe(Arc::new(SpecRecipe::V0Package(inner))) } ApiVersion::V0Platform => { - let inner = serde_yaml::from_value(value) - .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; + let inner = serde_yaml::from_str(yaml) + .map_err(|err| SerdeError::new(yaml.to_owned(), SerdeYamlError(err)))?; SpecFileData::Recipe(Arc::new(SpecRecipe::V0Platform(inner))) } + ApiVersion::V1Platform => { + let inner = serde_yaml::from_str(yaml) + .map_err(|err| SerdeError::new(yaml.to_owned(), SerdeYamlError(err)))?; + SpecFileData::Recipe(Arc::new(SpecRecipe::V1Platform(inner))) + } ApiVersion::V0Requirements => { - let requests: v0::Requirements = serde_yaml::from_value(value) - .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; + let requests: v0::Requirements = serde_yaml::from_str(yaml) + .map_err(|err| SerdeError::new(yaml.to_owned(), SerdeYamlError(err)))?; SpecFileData::Requests(requests) } }; @@ -851,10 +913,19 @@ impl FromYaml for Spec { .map_err(|err| SerdeError::new(yaml, SerdeYamlError(err)))?; Ok(Self::V0Package(inner)) } - ApiVersion::V0Requirements => { - // Reading a list of requests/requirement file is not - // supported here. But it might be in future. - unimplemented!() + v @ ApiVersion::V0Requirements | v @ ApiVersion::V1Platform => { + let api_version = v.to_string(); + let (start, end) = match yaml.find(&api_version) { + Some(start) => (Some(start), Some(start + api_version.len())), + None => (None, None), + }; + let error = Box::new(crate::Error::String(format!( + "api version '{api_version}' is not currently supported in this context" + ))) as Box; + Err(format_serde_error::SerdeError::new( + yaml, + (error, start, end), + )) } } } @@ -872,13 +943,19 @@ impl AsRef for Spec { } } -#[derive(Default, Deserialize, Serialize, Copy, Clone)] +#[derive(Default, Deserialize, Serialize, Copy, Clone, strum::Display)] pub enum ApiVersion { #[serde(rename = "v0/package")] #[default] + #[strum(to_string = "v0/package")] V0Package, #[serde(rename = "v0/platform")] + #[strum(to_string = "v0/platform")] V0Platform, + #[serde(rename = "v1/platform")] + #[strum(to_string = "v1/platform")] + V1Platform, #[serde(rename = "v0/requirements")] + #[strum(to_string = "v0/requirements")] V0Requirements, } diff --git a/crates/spk-schema/src/spec_test.rs b/crates/spk-schema/src/spec_test.rs index 3d34cf868a..e32b75a681 100644 --- a/crates/spk-schema/src/spec_test.rs +++ b/crates/spk-schema/src/spec_test.rs @@ -4,12 +4,13 @@ use rstest::rstest; use spk_schema_foundation::name::PkgName; -use spk_schema_foundation::option_map; use spk_schema_foundation::option_map::OptionMap; use spk_schema_foundation::spec_ops::HasVersion; +use spk_schema_foundation::{option_map, version}; use super::SpecTemplate; use crate::prelude::*; +use crate::template::{TemplateRenderConfig, TemplateSpec}; use crate::{Template, recipe}; #[rstest] @@ -316,19 +317,19 @@ fn test_get_build_requirements_pkg_in_variant_preserves_order() { #[rstest] fn test_template_error_message() { format_serde_error::never_color(); - static SPEC: &str = r#"pkg: my-package/{{ opt.version }} + static SPEC: &str = r#"pkg: my-package/{{ version }} sources: - git: https://downloads.testing/my-package/v{{ opt.typo }} "#; let tpl = SpecTemplate { + template_spec: TemplateSpec::from_single_version(version!("1.0.0")), + supported_versions: Default::default(), name: Some(PkgName::new("my-package").unwrap().to_owned()), file_path: "my-package.spk.yaml".into(), - versions: Default::default(), template: SPEC.into(), }; - let options = option_map! {"version" => "1.0.0"}; let err = tpl - .render(&options) + .render(TemplateRenderConfig::default()) .expect_err("expect template rendering to fail"); let expected = "Variable `opt.typo` not found"; let message = format!("{err:?}"); @@ -346,14 +347,18 @@ fn test_template_namespace_options() { format_serde_error::never_color(); static SPEC: &str = r#"pkg: mypackage/{{ opt.namespace.version }}"#; let tpl = SpecTemplate { + template_spec: TemplateSpec::from_single_version(version!("1.0.0")), + supported_versions: Default::default(), name: Some(PkgName::new("my-package").unwrap().to_owned()), file_path: "my-package.spk.yaml".into(), - versions: Default::default(), template: SPEC.into(), }; let options = option_map! {"namespace.version" => "1.0.0"}; let rendered_data = tpl - .render(&options) + .render(TemplateRenderConfig { + options, + ..Default::default() + }) .expect("template should render with sub-object access"); let recipe = rendered_data.into_recipe().unwrap(); assert_eq!(recipe.version().to_string(), "1.0.0"); diff --git a/crates/spk-schema/src/template.rs b/crates/spk-schema/src/template.rs index 953af71be9..fbea376046 100644 --- a/crates/spk-schema/src/template.rs +++ b/crates/spk-schema/src/template.rs @@ -2,11 +2,252 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::collections::HashMap; +use std::collections::{BTreeSet, HashMap}; use std::path::Path; +use std::str::FromStr; -use crate::foundation::option_map::OptionMap; -use crate::{Result, SpecFileData}; +use bracoxide::OxidizationError; +use bracoxide::tokenizer::TokenizationError; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Serialize}; +use spk_schema_foundation::option_map::OptionMap; +use spk_schema_foundation::version::Version; + +use crate::{Error, Result, SpecFileData}; + +/// A recipe template for building multiple versions of a package. +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct TemplateSpec { + /// Describes the versions that this template can produce. + /// + /// If none are specified, the package must have a single, hard-coded version + /// that can be parsed. + pub versions: TemplateVersions, +} + +impl TemplateSpec { + /// Constructs a template specification that can only + /// be used to produce a single pre-defined version. + pub fn from_single_version(version: Version) -> Self { + Self { + versions: TemplateVersions { + in_spec: Some(version), + allowed: None, + discover: None, + }, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct TemplateVersions { + /// The version specified in the template's `pkg` field, if any. + /// + /// This field will also be empty if the value could not be parsed + /// as a version, which is common for templates as they typically + /// inject the value dynamically via `{{ version }}`. + #[serde(skip)] + pub in_spec: Option, + /// Manually specified version numbers, with support for brace expansion of ranges. + #[serde(default, rename = "static", skip_serializing_if = "Option::is_none")] + pub allowed: Option, + /// Automatically discovered versions + #[serde(default, skip_serializing_if = "Option::is_none")] + pub discover: Option, +} + +impl DiscoverVersions for TemplateVersions { + fn discover_versions(&self) -> Result> { + let mut versions = BTreeSet::new(); + if let Some(in_spec) = &self.in_spec { + versions.insert(in_spec.clone()); + } + if let Some(allowed) = self.allowed.as_ref() { + versions.extend(allowed.0.iter().cloned()); + } + if let Some(discover) = self.discover.as_ref() { + versions.extend(discover.discover_versions()?); + } + Ok(versions) + } +} + +/// A set of manually specified versions. +/// +/// When deserializing, this will accept either a single string or +/// list of strings, and each one can contain one or more brace +/// expansion patterns. +/// +/// ``` +/// let versions: spk_schema::template::OrderedVersionSet = serde_yaml::from_str(r#"['1.0.{1..5}', '1.1.{1..3}']"#).unwrap(); +/// assert_eq!(versions, spk_schema::template::OrderedVersionSet(std::collections::BTreeSet::from_iter(vec![ +/// spk_schema::version!("1.0.1"), +/// spk_schema::version!("1.0.2"), +/// spk_schema::version!("1.0.3"), +/// spk_schema::version!("1.0.4"), +/// spk_schema::version!("1.0.5"), +/// spk_schema::version!("1.1.1"), +/// spk_schema::version!("1.1.2"), +/// spk_schema::version!("1.1.3"), +/// ]))); +/// ``` +#[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd, Default)] +pub struct OrderedVersionSet(pub BTreeSet); + +impl<'de> serde::de::Deserialize<'de> for OrderedVersionSet { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::de::Deserializer<'de>, + { + struct OrderedVersionSetVisitor; + + impl<'de> serde::de::Visitor<'de> for OrderedVersionSetVisitor { + type Value = OrderedVersionSet; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter + .write_str("a single or list of version numbers with optional brace expansions") + } + + fn visit_str(self, v: &str) -> std::result::Result + where + E: serde::de::Error, + { + let mut versions = BTreeSet::new(); + let expand_result = bracoxide::bracoxidize(v); + let expanded = match expand_result { + Ok(expanded) => expanded, + Err(OxidizationError::TokenizationError(TokenizationError::NoBraces)) + | Err(OxidizationError::TokenizationError(TokenizationError::EmptyContent)) + | Err(OxidizationError::TokenizationError( + TokenizationError::FormatNotSupported, + )) => { + vec![v.to_owned()] + } + Err(err) => { + return Err(serde::de::Error::custom(format!( + "invalid brace expansion: {err:?}" + ))); + } + }; + for version in expanded { + let parsed = Version::from_str(&version).map_err(|err| { + serde::de::Error::custom(format!( + "bad brace expansion or invalid version '{version}': {err}" + )) + })?; + versions.insert(parsed); + } + Ok(OrderedVersionSet(versions)) + } + + fn visit_seq(self, mut seq: A) -> std::result::Result + where + A: serde::de::SeqAccess<'de>, + { + let mut versions = BTreeSet::new(); + while let Some(version_expr) = seq.next_element()? { + versions.append(&mut OrderedVersionSetVisitor.visit_str(version_expr)?.0); + } + Ok(OrderedVersionSet(versions)) + } + } + + deserializer.deserialize_any(OrderedVersionSetVisitor) + } +} + +impl serde::Serialize for OrderedVersionSet { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: serde::Serializer, + { + if self.0.len() == 1 { + serializer.serialize_str(&self.0.iter().next().unwrap().to_string()) + } else { + let mut seq = serializer.serialize_seq(Some(self.0.len()))?; + for version in &self.0 { + seq.serialize_element(version)?; + } + seq.end() + } + } +} + +/// The strategy for discovering versions. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +#[enum_dispatch::enum_dispatch(DiscoverVersions)] +pub enum DiscoverStrategy { + GitTags(GitTagsDiscovery), +} + +/// Discover versions from git tags. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GitTagsDiscovery { + #[serde(with = "serde_regex")] + pub git_tags: Vec, + pub url: url::Url, + #[serde(default, with = "serde_regex", skip_serializing_if = "Vec::is_empty")] + pub extract: Vec, +} + +impl DiscoverVersions for GitTagsDiscovery { + fn discover_versions(&self) -> Result> { + let url = &self.url; + + let output = std::process::Command::new("git") + .args(["ls-remote", "--tags", "--quiet", "--refs"]) + .arg(url.as_str()) + .output() + .map_err(Error::GitCommandFailed)?; + + if !output.status.success() { + return Err(Error::GitCommandExited( + output.status.code().unwrap_or(1), + String::from_utf8_lossy(&output.stderr).to_string(), + )); + } + + let stdout = String::from_utf8_lossy(&output.stdout); + let refs = stdout + .lines() + // git ls-remote --tags outputs in the form OID refs/tags/ + .filter_map(|line| { + line.split_once('\t') + .and_then(|(_oid, tag)| tag.strip_prefix("refs/tags/")) + }) + // ensure that we only retain tags that match the specified patterns + .filter(|ref_name| { + self.git_tags.is_empty() || self.git_tags.iter().any(|re| re.is_match(ref_name)) + }) + // extract any specified capture group + .filter_map(|ref_name| { + if self.extract.is_empty() { + return Some(ref_name); + } + for pattern in self.extract.iter() { + let Some(groups) = pattern.captures(ref_name) else { + continue; + }; + if let Some(extracted) = groups.get(1) { + return Some(extracted.as_str()); + } + } + None + }) + .map(|v| Version::from_str(v).map_err(Error::FailedToParseTagAsVersion)); + + refs.collect() + } +} + +#[enum_dispatch::enum_dispatch] +pub trait DiscoverVersions { + fn discover_versions(&self) -> Result>; +} /// Can be rendered into a recipe. #[enum_dispatch::enum_dispatch] @@ -14,8 +255,14 @@ pub trait Template: Sized { /// Identify the location of this template on disk fn file_path(&self) -> &Path; - /// Render this template with the provided values. - fn render(&self, options: &OptionMap) -> Result; + /// Render this template to a string with the provided values. + fn render_to_string(&self, data: TemplateRenderConfig) -> Result; + + /// Render this template with the provided values and parse the output. + fn render(&self, data: TemplateRenderConfig) -> Result { + let rendered = self.render_to_string(data)?; + SpecFileData::from_yaml(rendered) + } } pub trait TemplateExt: Template { @@ -23,21 +270,92 @@ pub trait TemplateExt: Template { fn from_file(path: &Path) -> Result; } +/// Used to configure aspects of how a template will be rendered. +#[derive(Debug, Default, Clone)] +pub struct TemplateRenderConfig { + /// The version of the package to build. + /// + /// Exposed via the `version` variable in templates. + /// + /// If given, this version must be allowed by the template + /// and will be validated. If not given, the template must + /// either have a clear default or only be setup to build + /// a single version. + pub version: Option, + /// Additional options for the template rendering. + /// + /// These are exposed as the `opt` variable in templates + pub options: OptionMap, + /// Sets environment variables for the template data. + /// + /// These values will override any that are actually in the environment + /// when rendering. Exposed via the `env` variable in templates. + pub environment: HashMap, +} + +impl TemplateRenderConfig { + /// Sets the package version for the template data. + pub fn with_version(mut self, version: Version) -> Self { + self.version = Some(version); + self + } + + /// Sets additional options for the template data. + pub fn with_options(mut self, opt: I) -> Self + where + OptionMap: Extend, + I: IntoIterator, + { + self.options.extend(opt); + self + } + + /// Sets environment variables for the template data. + /// + /// These values will override any that are actually in the environment + /// when rendering. + pub fn with_environment_vars(mut self, env: I) -> Self + where + HashMap: Extend, + I: IntoIterator, + { + self.environment.extend(env); + self + } +} + /// The structured data that should be made available /// when rendering spk templates into recipes #[derive(serde::Serialize, Debug, Clone)] pub struct TemplateData { /// Information about the release of spk being used spk: SpkInfo, + /// The version of the package being built + version: Version, /// The option values for this template, expanded /// from an option map so that namespaced options - /// like `python.abi` actual live under the `python` + /// like `python.abi` actually live under the `python` /// field rather than as a field with a '.' in the name opt: serde_yaml::Mapping, /// Environment variable data for the current process env: HashMap, } +impl TemplateData { + /// Create the set of templating data for the current process and options + pub fn new(version: Version, options: OptionMap, mut env: HashMap) -> Self { + for (k, v) in std::env::vars() { + env.entry(k).or_insert(v); + } + TemplateData { + spk: SpkInfo::default(), + version, + opt: options.into_yaml_value_expanded(), + env, + } + } +} + /// The structured data that should be made available /// when rendering spk templates into recipes #[derive(serde::Serialize, Debug, Clone)] @@ -53,13 +371,64 @@ impl Default for SpkInfo { } } -impl TemplateData { - /// Create the set of templating data for the current process and options - pub fn new(options: &OptionMap) -> Self { - TemplateData { - spk: SpkInfo::default(), - opt: options.to_yaml_value_expanded(), - env: std::env::vars().collect(), +mod serde_regex { + use std::str::FromStr; + + use serde::Serialize; + use serde::ser::SerializeSeq; + + struct RegexVisitor; + + impl<'de> serde::de::Visitor<'de> for RegexVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a single or array of regular expression strings") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + regex::Regex::from_str(v) + .map_err(serde::de::Error::custom) + .map(|regex| vec![regex]) + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let mut regexes = Vec::with_capacity(seq.size_hint().unwrap_or(0)); + while let Some(regex) = seq.next_element()? { + regexes.push(regex::Regex::from_str(regex).map_err(serde::de::Error::custom)?); + } + Ok(regexes) + } + } + + pub fn deserialize<'de, D>(deserializer: D) -> std::result::Result, D::Error> + where + D: serde::de::Deserializer<'de>, + { + deserializer.deserialize_any(RegexVisitor) + } + + pub fn serialize( + value: &[regex::Regex], + serializer: S, + ) -> std::result::Result + where + S: serde::ser::Serializer, + { + if value.len() == 1 { + value[0].as_str().serialize(serializer) + } else { + let mut seq = serializer.serialize_seq(Some(value.len()))?; + for regex in value { + seq.serialize_element(regex.as_str())?; + } + seq.end() } } } diff --git a/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs b/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs index e1d99fe59b..a7ad228a21 100644 --- a/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs +++ b/crates/spk-schema/src/v0/embedded_recipe_spec_test.rs @@ -9,7 +9,6 @@ use spk_schema_foundation::ident_component::Component; use crate::foundation::FromYaml; use crate::foundation::fixtures::*; -use crate::foundation::option_map::OptionMap; use crate::spec::SpecTemplate; use crate::v0::EmbeddedRecipeSpec; use crate::{Opt, Recipe, SourceSpec, Template, TemplateExt}; @@ -42,7 +41,7 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { let spec = SpecTemplate::from_file(&spec_file) .unwrap() - .render(&OptionMap::default()) + .render(crate::template::TemplateRenderConfig::default()) .unwrap(); let crate::Spec::V0Package(recipe) = spec .into_recipe() diff --git a/crates/spk-schema/src/v0/package_spec_test.rs b/crates/spk-schema/src/v0/package_spec_test.rs index a98236670f..0c0ccae8df 100644 --- a/crates/spk-schema/src/v0/package_spec_test.rs +++ b/crates/spk-schema/src/v0/package_spec_test.rs @@ -36,7 +36,7 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { let spec = SpecTemplate::from_file(&spec_file) .unwrap() - .render(&OptionMap::default()) + .render(crate::template::TemplateRenderConfig::default()) .unwrap(); let crate::Spec::V0Package(recipe) = spec .into_recipe() diff --git a/crates/spk-schema/src/v0/recipe_spec_test.rs b/crates/spk-schema/src/v0/recipe_spec_test.rs index 791f987340..49b6fdce82 100644 --- a/crates/spk-schema/src/v0/recipe_spec_test.rs +++ b/crates/spk-schema/src/v0/recipe_spec_test.rs @@ -42,7 +42,7 @@ fn test_sources_relative_to_spec_file(tmpdir: tempfile::TempDir) { let spec = SpecTemplate::from_file(&spec_file) .unwrap() - .render(&OptionMap::default()) + .render(Default::default()) .unwrap(); let crate::Spec::V0Package(recipe) = spec .into_recipe() diff --git a/crates/spk-schema/src/v1/mod.rs b/crates/spk-schema/src/v1/mod.rs index a5fddb929b..9f8fcf2d57 100644 --- a/crates/spk-schema/src/v1/mod.rs +++ b/crates/spk-schema/src/v1/mod.rs @@ -4,4 +4,10 @@ mod platform; -pub use platform::Platform; +pub use platform::{ + Override, + Platform, + PlatformPkgRequirement, + PlatformRequirement, + PlatformVarRequirement, +}; diff --git a/crates/spk-schema/src/v1/platform.rs b/crates/spk-schema/src/v1/platform.rs index 23a2d80b38..a561c58f79 100644 --- a/crates/spk-schema/src/v1/platform.rs +++ b/crates/spk-schema/src/v1/platform.rs @@ -6,11 +6,11 @@ use std::borrow::Cow; use std::path::Path; use std::sync::Arc; +use serde::de::value::MapAccessDeserializer; use serde::{Deserialize, Serialize}; use spk_schema_foundation::IsDefault; use spk_schema_foundation::ident::{ InclusionPolicy, - NameAndValue, PinnableRequest, PkgRequest, PkgRequestOptions, @@ -23,8 +23,8 @@ use spk_schema_foundation::ident::{ }; use spk_schema_foundation::ident_build::{Build, BuildId}; use spk_schema_foundation::ident_component::Component; -use spk_schema_foundation::name::{OptName, PkgName}; -use spk_schema_foundation::option_map::{HOST_OPTIONS, OptionMap}; +use spk_schema_foundation::name::{OptName, OptNameBuf, PkgName, PkgNameBuf}; +use spk_schema_foundation::option_map::{HOST_OPTIONS, OptionMap, Stringified}; use spk_schema_foundation::spec_ops::{HasVersion, Named, Versioned}; use spk_schema_foundation::version::Version; use spk_schema_foundation::version_range::VersionFilter; @@ -285,7 +285,7 @@ fn apply_inherit_from_base_component( } } -#[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize)] #[serde(untagged)] pub enum PlatformRequirement { Pkg(PlatformPkgRequirement), @@ -319,27 +319,70 @@ impl PlatformRequirement { } } +impl<'de> Deserialize<'de> for PlatformRequirement { + fn deserialize(deserializer: D) -> std::result::Result + where + D: serde::Deserializer<'de>, + { + struct PlatformRequirementVisitor; + + impl<'de> serde::de::Visitor<'de> for PlatformRequirementVisitor { + type Value = PlatformRequirement; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a pkg or var requirement") + } + + fn visit_map(self, map: A) -> std::result::Result + where + A: serde::de::MapAccess<'de>, + { + let mut peekable = serde_peekable::PeekableMapAccess::from(map); + let first_key = peekable.peek_key::()?; + let Some(first_key) = first_key else { + return Err(serde::de::Error::missing_field("pkg or var")); + }; + match first_key.as_ref() { + "pkg" => Ok(PlatformRequirement::Pkg(Deserialize::deserialize( + MapAccessDeserializer::new(peekable), + )?)), + "var" => Ok(PlatformRequirement::Var(Deserialize::deserialize( + MapAccessDeserializer::new(peekable), + )?)), + _ => Err(serde::de::Error::custom( + "expected 'pkg' or 'var' as the first key", + )), + } + } + } + + deserializer.deserialize_any(PlatformRequirementVisitor) + } +} + #[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct PlatformPkgRequirement { - pkg: VersionIdent, + pub pkg: PkgNameBuf, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub build: Option, #[serde( default, with = "value_or_false", skip_serializing_if = "Option::is_none" )] - at_build: Option>, + pub at_build: Option>, #[serde( default, with = "value_or_false", skip_serializing_if = "Option::is_none" )] - at_runtime: Option>, + pub at_runtime: Option>, } impl Named for PlatformPkgRequirement { fn name(&self) -> &OptName { - self.pkg.name().as_opt_name() + self.pkg.as_opt_name() } } @@ -367,7 +410,7 @@ impl PlatformPkgRequirement { .insert_or_replace(PinnableRequest::Pkg(PkgRequest { pkg: RangeIdent { repository_name: None, - name: self.pkg.name().to_owned(), + name: self.pkg.clone(), version: v.clone(), components: Default::default(), build: None, @@ -395,7 +438,7 @@ impl PlatformPkgRequirement { .insert_or_replace(PinnableRequest::Pkg(PkgRequest { pkg: RangeIdent { repository_name: None, - name: self.pkg.name().to_owned(), + name: self.pkg.clone(), version: v.clone(), components: Default::default(), build: None, @@ -417,24 +460,24 @@ impl PlatformPkgRequirement { #[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] #[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct PlatformVarRequirement { - var: NameAndValue, + pub var: OptNameBuf, #[serde( default, with = "value_or_false", skip_serializing_if = "Option::is_none" )] - at_build: Option>, + pub at_build: Option>, #[serde( default, with = "value_or_false", skip_serializing_if = "Option::is_none" )] - at_runtime: Option>, + pub at_runtime: Option>, } impl Named for PlatformVarRequirement { fn name(&self) -> &OptName { - &self.var.0 + &self.var } } @@ -464,7 +507,7 @@ impl PlatformVarRequirement { build_component .requirements .insert_or_replace(PinnableRequest::Var(VarRequest { - var: self.var.0.clone(), + var: self.var.clone(), value: spk_schema_foundation::ident::PinnableValue::Pinned(Arc::from( v.as_str(), )), @@ -484,7 +527,7 @@ impl PlatformVarRequirement { runtime_component .requirements .insert_or_replace(PinnableRequest::Var(VarRequest { - var: self.var.0.clone(), + var: self.var.clone(), value: spk_schema_foundation::ident::PinnableValue::Pinned(Arc::from( v.as_str(), )), @@ -497,6 +540,61 @@ impl PlatformVarRequirement { } } +/// Marks a package to be buildable in a platform. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Deserialize, Serialize)] +pub struct PlatformPkgBuildConfig { + /// The version to build when building this platform + #[serde(deserialize_with = "deserialize_version_allow_number")] + pub version: Version, +} + +fn deserialize_version_allow_number<'de, D>( + deserializer: D, +) -> std::result::Result +where + D: serde::de::Deserializer<'de>, +{ + struct VersionVisitor; + + impl<'de> serde::de::Visitor<'de> for VersionVisitor { + type Value = Version; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("a string or number representing a version") + } + + fn visit_i64(self, v: i64) -> std::result::Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + + fn visit_u64(self, v: u64) -> std::result::Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + + fn visit_f64(self, v: f64) -> std::result::Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + + fn visit_str(self, v: &str) -> std::result::Result + where + E: serde::de::Error, + { + v.parse().map_err(serde::de::Error::custom) + } + } + + deserializer.deserialize_any(VersionVisitor) +} + /// Overrides the value of some request within a platform #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub enum Override { @@ -542,6 +640,27 @@ mod value_or_false { )) } + fn visit_i64(self, v: i64) -> Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + + fn visit_u64(self, v: u64) -> Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + + fn visit_f64(self, v: f64) -> Result + where + E: serde::de::Error, + { + self.visit_str(&v.to_string()) + } + fn visit_str(self, v: &str) -> Result where E: serde::de::Error, diff --git a/crates/spk-schema/src/v1/platform_test.rs b/crates/spk-schema/src/v1/platform_test.rs index b63de56437..517abf3d23 100644 --- a/crates/spk-schema/src/v1/platform_test.rs +++ b/crates/spk-schema/src/v1/platform_test.rs @@ -147,7 +147,7 @@ fn test_platform_single_inheritance() { platform: base/1.0.0 requirements: - pkg: inherit-me - atRuntime: 1.0.0 + atRuntime: 1.0 "#, ) .unwrap(); @@ -306,3 +306,68 @@ fn test_platform_inheritance_with_override_and_removal() { assert_requirements!(build:Run len 2); assert_requirements!(build:Build len 3); } + +#[rstest] +fn test_platform_requirement_deserialize_numbers() { + // test that any version-like value can be deserialized + // in platform requirements. Notably floating point and + // integer values + + let yaml = r#" + platform: test-platform + requirements: + - pkg: test1 + atRuntime: 1.0.0 + - pkg: test2 + atRuntime: 1.0 + - pkg: test3 + atRuntime: 1 + "#; + + let spec: Platform = serde_yaml::from_str(yaml).unwrap(); + let build = spec.generate_binary_build(&option_map! {}, &()).unwrap(); + + assert_requirements!(build:Run contains "test1/1.0.0"); + assert_requirements!(build:Run contains "test2/1.0.0"); + assert_requirements!(build:Run contains "test3/1.0.0"); + assert_requirements!(build:Run len 3); +} + +#[rstest] +fn test_platform_requirement_deserialize_vfx_reference() { + let yaml = r#" +api: v1/platform +platform: vfx-reference/2024 +requirements: + - pkg: gcc + atBuild: 11.2.1 + - pkg: python + build: + version: 3.11 + atBuild: 3.11.0 + atRuntime: 3.11 + "#; + + let res: Result = serde_yaml::from_str(yaml); + match res { + Ok(_) => {} + Err(e) => panic!("Deserialization failed with: {}", e), + } +} + +#[rstest] +fn test_platform_requirement_deserialize_310() { + let yaml = r#" +api: v1/platform +platform: vfx-reference/2024 +requirements: + - pkg: python + build: + version: 3.10 + atBuild: 3.10 + atRuntime: 3.10 + "#; + + let res: Result = serde_yaml::from_str(yaml); + println!("{:#?}", res.unwrap().requirements); +} diff --git a/crates/spk-solve/crates/graph/src/graph.rs b/crates/spk-solve/crates/graph/src/graph.rs index 2692419680..bec0195a4a 100644 --- a/crates/spk-solve/crates/graph/src/graph.rs +++ b/crates/spk-solve/crates/graph/src/graph.rs @@ -49,6 +49,7 @@ use spk_schema::{ }; use spk_solve_package_iterator::{PackageIterator, PromotionPatterns}; use spk_solve_solution::{PackageSource, Solution}; +use spk_storage::RepositoryHandle; use thiserror::Error; use crate::GetMergedRequestError; @@ -176,7 +177,7 @@ impl FormatChange for Change { // requested and added in the same state during a solve. // We can use their PackageSource data to find what // requested them. - PackageSource::BuildFromSource { recipe } => { + PackageSource::BuildFromSource { recipe, .. } => { vec![ RequestedBy::PackageVersion(recipe.ident().clone()) .to_string(), @@ -285,11 +286,13 @@ impl<'state> DecisionBuilder<'state, '_> { /// can be included in the final solution. pub fn build_package( self, + repo: &Arc, recipe: &Arc, spec: &Arc, ) -> crate::Result { let generate_changes = || -> crate::Result> { let mut changes = vec![Change::SetPackageBuild(Box::new(SetPackageBuild::new( + Arc::clone(repo), Arc::clone(spec), Arc::clone(recipe), )))]; @@ -1168,10 +1171,10 @@ pub struct SetPackageBuild { } impl SetPackageBuild { - pub fn new(spec: Arc, recipe: Arc) -> Self { + pub fn new(repo: Arc, spec: Arc, recipe: Arc) -> Self { SetPackageBuild { spec, - source: PackageSource::BuildFromSource { recipe }, + source: PackageSource::BuildFromSource { recipe, repo }, } } diff --git a/crates/spk-solve/crates/graph/src/graph_test.rs b/crates/spk-solve/crates/graph/src/graph_test.rs index b517bfe2f8..64ee8c3ac9 100644 --- a/crates/spk-solve/crates/graph/src/graph_test.rs +++ b/crates/spk-solve/crates/graph/src/graph_test.rs @@ -11,6 +11,7 @@ use spk_schema::foundation::name::PkgName; use spk_schema::foundation::{opt_name, option_map}; use spk_schema::{recipe, spec}; use spk_solve_solution::PackageSource; +use spk_storage::RepositoryHandle; use super::DecisionBuilder; use crate::{Decision, graph}; @@ -28,12 +29,13 @@ fn test_resolve_build_same_result() { let build_spec = spec!({"pkg": "test/1.0.0/3I42H3S6"}); let build_spec = Arc::new(build_spec); let source = PackageSource::SpkInternalTest; + let repo = Arc::new(RepositoryHandle::new_mem()); let resolve = Decision::builder(&base) .resolve_package(&build_spec, source) .unwrap(); let build = Decision::builder(&base) - .build_package(&recipe, &build_spec) + .build_package(&repo, &recipe, &build_spec) .unwrap(); let with_binary = resolve.apply(&base); @@ -126,9 +128,10 @@ fn test_request_default_component() { .contains(&Component::default_for_run()), "default component should be injected when none specified" ); + let repo = Arc::new(RepositoryHandle::new_mem()); let build_state = DecisionBuilder::new(&base) - .build_package(&recipe, &spec) + .build_package(&repo, &recipe, &spec) .unwrap() .apply(&base); let request = build_state diff --git a/crates/spk-solve/crates/solution/src/solution.rs b/crates/spk-solve/crates/solution/src/solution.rs index b1d28d7c3a..3cde92b1b2 100644 --- a/crates/spk-solve/crates/solution/src/solution.rs +++ b/crates/spk-solve/crates/solution/src/solution.rs @@ -56,6 +56,8 @@ pub enum PackageSource { }, /// The package needs to be build from the given recipe. BuildFromSource { + /// the actual repository that this source spec was loaded from, if any + repo: Arc, /// The recipe that this package is to be built from. recipe: Arc, }, @@ -76,7 +78,7 @@ impl PackageSource { pub async fn read_recipe(&self, ident: &VersionIdent) -> Result> { match self { - PackageSource::BuildFromSource { recipe } => Ok(Arc::clone(recipe)), + PackageSource::BuildFromSource { recipe, .. } => Ok(Arc::clone(recipe)), PackageSource::Repository { repo, .. } => Ok(repo.read_recipe(ident).await?), PackageSource::Embedded { .. } => { // TODO: what are the implications of this? @@ -107,7 +109,7 @@ impl Ord for PackageSource { (SpkInternalTest, Embedded { .. }) => Ordering::Less, (Embedded { .. } | SpkInternalTest, BuildFromSource { .. }) => Ordering::Less, (BuildFromSource { .. }, Embedded { .. } | SpkInternalTest) => Ordering::Greater, - (BuildFromSource { recipe: this }, BuildFromSource { recipe: other }) => { + (BuildFromSource { recipe: this, .. }, BuildFromSource { recipe: other, .. }) => { this.ident().cmp(other.ident()) } (SpkInternalTest, SpkInternalTest) => Ordering::Equal, @@ -252,14 +254,8 @@ impl SolvedRequest { // available here yet. None } - PackageSource::BuildFromSource { .. } => { - // Packages that need building are not in a repo yet. - None - } - PackageSource::Repository { - repo, - components: _, - } => Some(repo.name().into()), + PackageSource::BuildFromSource { repo, .. } + | PackageSource::Repository { repo, .. } => Some(repo.name().into()), PackageSource::SpkInternalTest => None, } } @@ -276,7 +272,7 @@ impl std::fmt::Debug for SolvedRequest { PackageSource::Repository { repo, .. } => { format!("Repository={}", repo.name()) } - PackageSource::BuildFromSource { recipe } => { + PackageSource::BuildFromSource { recipe, .. } => { format!("BuildFromSource={}", recipe.ident()) } PackageSource::Embedded { parent, .. } => format!("Embedded={parent}"), diff --git a/crates/spk-solve/crates/validation/src/validators/pkg_request.rs b/crates/spk-solve/crates/validation/src/validators/pkg_request.rs index 0a1cbaa595..c9aa3b4017 100644 --- a/crates/spk-solve/crates/validation/src/validators/pkg_request.rs +++ b/crates/spk-solve/crates/validation/src/validators/pkg_request.rs @@ -84,7 +84,14 @@ impl ValidatorT for PkgRequestValidator { if let Some(rn) = &request.pkg.repository_name { // If the request names a repository, then the source has to match. match source { - PackageSource::Repository { repo, .. } if repo.name() != rn => { + PackageSource::BuildFromSource { repo, .. } + | PackageSource::Repository { repo, .. } + if repo.name() == rn => + { + // the only allowed cases + } + PackageSource::BuildFromSource { repo, .. } + | PackageSource::Repository { repo, .. } => { return Ok(Compatibility::Incompatible( IncompatibleReason::PackageRepoMismatch( PackageRepoProblem::WrongSourceRepository { @@ -94,7 +101,6 @@ impl ValidatorT for PkgRequestValidator { ), )); } - PackageSource::Repository { .. } => {} // okay PackageSource::Embedded { parent, .. } => { // TODO: from the right repo still? return Ok(Compatibility::Incompatible( @@ -105,14 +111,6 @@ impl ValidatorT for PkgRequestValidator { ), )); } - PackageSource::BuildFromSource { .. } => { - // TODO: from the right repo still? - return Ok(Compatibility::Incompatible( - IncompatibleReason::PackageRepoMismatch( - PackageRepoProblem::FromRecipeFromWrongRepository, - ), - )); - } PackageSource::SpkInternalTest => { return Ok(Compatibility::Incompatible( IncompatibleReason::PackageRepoMismatch(PackageRepoProblem::InternalTest), diff --git a/crates/spk-solve/src/solvers/resolvo/mod.rs b/crates/spk-solve/src/solvers/resolvo/mod.rs index cc0526bc44..5eb38d7648 100644 --- a/crates/spk-solve/src/solvers/resolvo/mod.rs +++ b/crates/spk-solve/src/solvers/resolvo/mod.rs @@ -319,6 +319,7 @@ impl Solver { match ident.build() { spk_schema::ident_build::Build::Source if *requires_build_from_source => { PackageSource::BuildFromSource { + repo: Arc::clone(repo), recipe: repo.read_recipe(&ident.clone().to_version_ident()).await?, } } diff --git a/crates/spk-solve/src/solvers/step/solver.rs b/crates/spk-solve/src/solvers/step/solver.rs index b84b56e7d1..5be7088468 100644 --- a/crates/spk-solve/src/solvers/step/solver.rs +++ b/crates/spk-solve/src/solvers/step/solver.rs @@ -900,7 +900,23 @@ impl Solver { } res => res?, }; + + let PackageSource::Repository { repo, .. } = &source else { + notes.push(Note::SkipPackageNote(Box::new( + SkipPackageNote::new_from_message( + spec.ident().to_any_ident(), + "building from source not possible: src package was not from a repository" + ), + ))); + debug_assert!( + false, + "this should not be possible, where else would this source package be from?" + ); + continue; + }; + let new_source = PackageSource::BuildFromSource { + repo: Arc::clone(repo), recipe: Arc::clone(&recipe), }; @@ -918,7 +934,7 @@ impl Solver { match Decision::builder(&node.state) .with_components(&request.pkg.components) - .build_package(&recipe, &new_spec) + .build_package(repo, &recipe, &new_spec) { Ok(decision) => decision, Err(err) => { diff --git a/crates/spk-storage/Cargo.toml b/crates/spk-storage/Cargo.toml index 7d9bec1757..b892f77719 100644 --- a/crates/spk-storage/Cargo.toml +++ b/crates/spk-storage/Cargo.toml @@ -47,6 +47,7 @@ serde_json = { workspace = true } serde_yaml = { workspace = true } spfs = { workspace = true } spk-schema = { workspace = true } +spk-workspace = { workspace = true } sys-info = "0.9.0" tar = "0.4.30" tempfile = { workspace = true } @@ -57,3 +58,6 @@ tracing-subscriber = { workspace = true } ulid = { workspace = true } url = "2.2" variantly = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/spk-storage/src/error.rs b/crates/spk-storage/src/error.rs index 61cf1eeacb..5b70a30ff2 100644 --- a/crates/spk-storage/src/error.rs +++ b/crates/spk-storage/src/error.rs @@ -63,6 +63,12 @@ pub enum Error { #[error(transparent)] #[diagnostic(forward(0))] SpkSpecError(Box), + #[error(transparent)] + #[diagnostic(forward(0))] + SpkWorkspaceFromPathError(#[from] spk_workspace::error::FromPathError), + #[error(transparent)] + #[diagnostic(forward(0))] + SpkWorkspaceBuildError(#[from] spk_workspace::error::BuildError), #[error("No disk usage: version '{0}' not found")] DiskUsageVersionNotFound(String), #[error("No disk usage: build '{0}' not found")] diff --git a/crates/spk-storage/src/lib.rs b/crates/spk-storage/src/lib.rs index 200f4bfb12..43cb5841e4 100644 --- a/crates/spk-storage/src/lib.rs +++ b/crates/spk-storage/src/lib.rs @@ -31,6 +31,7 @@ pub use storage::{ RuntimeRepository, SpfsRepository, Storage, + WorkspaceRepository, export_package, find_path_providers, inject_path_repo_into_spfs_config, diff --git a/crates/spk-storage/src/storage/handle.rs b/crates/spk-storage/src/storage/handle.rs index 3c332c717c..566ecbe1b4 100644 --- a/crates/spk-storage/src/storage/handle.rs +++ b/crates/spk-storage/src/storage/handle.rs @@ -6,7 +6,8 @@ use spk_schema::{Spec, SpecRecipe}; use super::Repository; -type Handle = dyn Repository; +/// A type alias for a boxed repository trait object. +pub type Handle = dyn Repository; #[derive(Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[allow(clippy::large_enum_variant)] @@ -14,6 +15,7 @@ pub enum RepositoryHandle { SPFS(super::SpfsRepository), Mem(super::MemRepository), Runtime(super::RuntimeRepository), + Workspace(super::WorkspaceRepository), } impl RepositoryHandle { @@ -47,6 +49,7 @@ impl RepositoryHandle { Self::SPFS(repo) => Box::new(repo), Self::Mem(repo) => Box::new(repo), Self::Runtime(repo) => Box::new(repo), + Self::Workspace(repo) => Box::new(repo), } } } @@ -59,6 +62,7 @@ impl std::ops::Deref for RepositoryHandle { RepositoryHandle::SPFS(repo) => repo, RepositoryHandle::Mem(repo) => repo, RepositoryHandle::Runtime(repo) => repo, + RepositoryHandle::Workspace(repo) => repo, } } } @@ -69,6 +73,7 @@ impl std::ops::DerefMut for RepositoryHandle { RepositoryHandle::SPFS(repo) => repo, RepositoryHandle::Mem(repo) => repo, RepositoryHandle::Runtime(repo) => repo, + RepositoryHandle::Workspace(repo) => repo, } } } @@ -90,3 +95,9 @@ impl From for RepositoryHandle { RepositoryHandle::Runtime(repo) } } + +impl From for RepositoryHandle { + fn from(repo: super::WorkspaceRepository) -> Self { + RepositoryHandle::Workspace(repo) + } +} diff --git a/crates/spk-storage/src/storage/mod.rs b/crates/spk-storage/src/storage/mod.rs index 2b3d7240d4..7ce8105b0d 100644 --- a/crates/spk-storage/src/storage/mod.rs +++ b/crates/spk-storage/src/storage/mod.rs @@ -8,12 +8,14 @@ mod mem; mod repository; mod runtime; mod spfs; +mod workspace; pub use archive::export_package; pub use handle::RepositoryHandle; pub use mem::MemRepository; pub use repository::{CachePolicy, Repository, Storage}; pub use runtime::{RuntimeRepository, find_path_providers, pretty_print_filepath}; +pub use workspace::WorkspaceRepository; pub use self::spfs::{ NameAndRepository, diff --git a/crates/spk-storage/src/storage/workspace.rs b/crates/spk-storage/src/storage/workspace.rs new file mode 100644 index 0000000000..8ba3c59e14 --- /dev/null +++ b/crates/spk-storage/src/storage/workspace.rs @@ -0,0 +1,317 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::Arc; + +use itertools::Itertools; +use spk_schema::foundation::ident_component::Component; +use spk_schema::foundation::name::{PkgName, PkgNameBuf, RepositoryName, RepositoryNameBuf}; +use spk_schema::foundation::version::Version; +use spk_schema::ident::ToAnyIdentWithoutBuild; +use spk_schema::ident_build::Build; +use spk_schema::template::DiscoverVersions; +use spk_schema::{BuildIdent, Recipe, Spec, SpecRecipe, Template, VersionIdent}; + +use super::Repository; +use super::repository::{PublishPolicy, Storage}; +use crate::{Error, Result}; + +#[cfg(test)] +#[path = "./workspace_test.rs"] +mod workspace_test; + +/// A repository that represents package build for +/// and from an [`spk_workspace::Workspace`]. +#[derive(Clone, Debug)] +pub struct WorkspaceRepository { + address: url::Url, + name: RepositoryNameBuf, + inner: spk_workspace::Workspace, +} + +impl std::hash::Hash for WorkspaceRepository { + fn hash(&self, state: &mut H) { + self.address.hash(state); + } +} + +impl Eq for WorkspaceRepository {} +impl PartialEq for WorkspaceRepository { + fn eq(&self, other: &Self) -> bool { + self.address == other.address + } +} + +impl Ord for WorkspaceRepository { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.address.cmp(&other.address) + } +} + +impl PartialOrd for WorkspaceRepository { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl WorkspaceRepository { + /// Build a workspace repository from its parts. + pub fn new( + root: &std::path::Path, + name: RepositoryNameBuf, + workspace: spk_workspace::Workspace, + ) -> Self { + let address = Self::address_from_root(root); + Self { + address, + name, + inner: workspace, + } + } + + /// Open a workspace repository from its root directory, using the default name. + /// + /// Panics if the workspace cannot be loaded at the given path. + #[cfg(test)] + pub fn open(root: &std::path::Path) -> Result { + // this function is not allowed outside of testing because it will + // panic if the workspace is invalid + let address = Self::address_from_root(root); + Ok(Self { + address, + name: "workspace".try_into()?, + inner: spk_workspace::Workspace::builder() + .load_from_dir(root)? + .build()?, + }) + } + + fn address_from_root(root: &std::path::Path) -> url::Url { + let address = format!("workspace://{}", root.display()); + match url::Url::parse(&address) { + Ok(a) => a, + Err(err) => { + tracing::error!( + "failed to create valid address for path {:?}: {:?}", + root, + err + ); + url::Url::parse(&format!("workspace://{}", root.to_string_lossy())) + .expect("Failed to create url from path (fallback)") + } + } + } +} + +impl std::ops::Deref for WorkspaceRepository { + type Target = spk_workspace::Workspace; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +#[async_trait::async_trait] +impl Storage for WorkspaceRepository { + type Recipe = SpecRecipe; + type Package = Spec; + + async fn get_concrete_package_builds(&self, pkg: &VersionIdent) -> Result> { + // assuming that this version was previously loaded via list_versions, + // we can present that a source build is available. + // TODO: it's not clear if this assumption will turn out dangerous, + // but we generally assume that the solver won't try to look for build + // of a package version that it doesn't know exists... + let mut builds = HashSet::new(); + builds.insert(pkg.to_build_ident(Build::Source)); + Ok(builds) + } + + async fn get_embedded_package_builds( + &self, + _pkg: &VersionIdent, + ) -> Result> { + // Can't publish packages to a workspace so there can't be any stubs + Ok(HashSet::default()) + } + + async fn publish_embed_stub_to_storage(&self, _spec: &Self::Package) -> Result<()> { + Err(Error::String( + "Cannot publish to a workspace repository".into(), + )) + } + + async fn publish_package_to_storage( + &self, + _package: &::Output, + _components: &HashMap, + ) -> Result<()> { + Err(Error::String( + "Cannot publish to a workspace repository".into(), + )) + } + + async fn publish_recipe_to_storage( + &self, + _spec: &Self::Recipe, + _publish_policy: PublishPolicy, + ) -> Result<()> { + Err(Error::String( + "Cannot publish to a workspace repository".into(), + )) + } + + async fn read_components_from_storage( + &self, + _pkg: &BuildIdent, + ) -> Result> { + Ok(HashMap::new()) + } + + async fn read_package_from_storage( + &self, + pkg: &BuildIdent, + ) -> Result::Output>> { + if !pkg.is_source() { + return Err(Error::PackageNotFound(Box::new( + pkg.clone().into_any_ident(), + ))); + } + + let mut candidates = Vec::new(); + for (name, tpl) in self.inner.iter() { + if name != pkg.name() { + continue; + } + let versions = tpl.discover_versions()?; + if versions.contains(pkg.version()) { + candidates.push(tpl); + } + } + if candidates.is_empty() { + return Err(Error::PackageNotFound(Box::new(pkg.to_any_ident()))); + } + if candidates.len() > 1 { + tracing::warn!( + "multiple viable recipes found in workspace for {pkg} [{}]", + candidates + .iter() + .map(|r| r.file_path().to_string_lossy()) + .join(", ") + ); + } + let rendered = candidates[0].render(spk_schema::template::TemplateRenderConfig { + version: Some(pkg.version().to_owned()), + ..Default::default() + })?; + let recipe = rendered.into_recipe().map_err(|err| { + Error::String(format!( + "Failed to convert rendered template into recipe: {err}" + )) + })?; + let build = recipe.generate_source_build( + candidates[0] + .file_path() + .parent() + .or(self.inner.root()) + .unwrap_or_else(|| std::path::Path::new(".")), + )?; + Ok(Arc::new(build)) + } + + async fn remove_embed_stub_from_storage(&self, _pkg: &BuildIdent) -> Result<()> { + Err(Error::String("Cannot modify a workspace repository".into())) + } + + async fn remove_package_from_storage(&self, _pkg: &BuildIdent) -> Result<()> { + Err(Error::String("Cannot modify a workspace repository".into())) + } +} + +#[async_trait::async_trait] +impl Repository for WorkspaceRepository { + fn address(&self) -> &url::Url { + &self.address + } + + fn name(&self) -> &RepositoryName { + &self.name + } + + async fn list_packages(&self) -> Result> { + let unique = self + .inner + .iter() + .map(|(name, _)| name.to_owned()) + .collect::>(); + Ok(unique.into_iter().collect()) + } + + async fn list_package_versions(&self, name: &PkgName) -> Result>>> { + let mut versions = HashSet::new(); + for (tpl_name, tpl) in self.inner.iter() { + if tpl_name != name { + continue; + } + let discovered = tpl.discover_versions()?; + versions.extend(discovered.into_iter().map(Arc::new)); + } + let mut sorted = versions.into_iter().collect::>(); + sorted.sort(); + Ok(Arc::new(sorted)) + } + + async fn list_build_components(&self, pkg: &BuildIdent) -> Result> { + if pkg.is_source() { + return Ok(vec![Component::Source]); + } + Err(Error::PackageNotFound(Box::new(pkg.to_any_ident()))) + } + + async fn read_embed_stub(&self, pkg: &BuildIdent) -> Result> { + Err(Error::PackageNotFound(Box::new(pkg.to_any_ident()))) + } + + async fn read_recipe(&self, pkg: &VersionIdent) -> Result> { + let mut candidates = Vec::new(); + + for (name, tpl) in self.inner.iter() { + if name != pkg.name() { + continue; + } + let versions = tpl.discover_versions()?; + if versions.contains(pkg.version()) { + candidates.push(tpl); + } + } + if candidates.is_empty() { + return Err(Error::PackageNotFound(Box::new( + pkg.to_any_ident_without_build(), + ))); + } + if candidates.len() > 1 { + tracing::warn!( + "multiple viable recipes found in workspace for {pkg} [{}]", + candidates + .iter() + .map(|r| r.file_path().to_string_lossy()) + .join(", ") + ); + } + let rendered = candidates[0].render(spk_schema::template::TemplateRenderConfig { + version: Some(pkg.version().to_owned()), + ..Default::default() + })?; + rendered.into_recipe().map_err(|err| { + Error::String(format!( + "Failed to convert rendered template into recipe: {err}" + )) + }) + } + + async fn remove_recipe(&self, _pkg: &VersionIdent) -> Result<()> { + Err(Error::String("Cannot modify a workspace repository".into())) + } +} diff --git a/crates/spk-storage/src/storage/workspace_test.rs b/crates/spk-storage/src/storage/workspace_test.rs new file mode 100644 index 0000000000..a0c63d5ce4 --- /dev/null +++ b/crates/spk-storage/src/storage/workspace_test.rs @@ -0,0 +1,235 @@ +// Copyright (c) Contributors to the SPK project. +// SPDX-License-Identifier: Apache-2.0 +// https://github.com/spkenv/spk + +use std::fs; + +use rstest::rstest; +use spk_schema::foundation::name::PkgNameBuf; +use spk_schema::ident_build::{Build, BuildId}; +use spk_schema::prelude::*; +use spk_schema::{VersionIdent, version}; +use spk_workspace::Workspace; +use tempfile::TempDir; + +use crate::storage::workspace::WorkspaceRepository; +use crate::{Repository, Storage}; + +struct TestWorkspace { + _temp_dir: TempDir, + repo: WorkspaceRepository, +} + +impl TestWorkspace { + fn new(files: &[(&str, &str)]) -> Self { + let temp_dir = tempfile::tempdir().unwrap(); + for (name, content) in files { + fs::write(temp_dir.path().join(name), content).unwrap(); + } + let workspace = Workspace::builder() + .with_root(temp_dir.path()) + .with_ignore_invalid_files(false) + .with_glob_pattern("*.spk.yaml") + .unwrap() + .build() + .unwrap(); + let repo = WorkspaceRepository::new( + temp_dir.path(), + "test-workspace".try_into().unwrap(), + workspace, + ); + Self { + _temp_dir: temp_dir, + repo, + } + } +} + +#[rstest] +#[tokio::test] +async fn test_list_package_versions_multiple_templates() { + let ws = TestWorkspace::new(&[ + ( + "pkg-a.spk.yaml", + r#" + template: + versions: + static: ["0.1.0"] + pkg: pkg-a/{{ version }} + "#, + ), + ( + "pkg-a.v2.spk.yaml", + r#" + template: + versions: + static: ["0.2.0"] + pkg: pkg-a/{{ version }} + "#, + ), + ]); + + let versions = ws + .repo + .list_package_versions(&"pkg-a".parse::().unwrap()) + .await + .unwrap(); + let versions = versions.iter().map(|v| (**v).clone()).collect::>(); + assert_eq!(versions, vec![version!("0.1.0"), version!("0.2.0")]); +} + +#[rstest] +#[tokio::test] +async fn test_list_package_versions_with_discovery_error() { + let ws = TestWorkspace::new(&[ + ( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a + template: + versions: + static: ["0.1.0"] + "#, + ), + ( + "pkg-a.v2.spk.yaml", + r#" + pkg: pkg-a + template: + versions: + discover: + gitTags: "v0.2.*" + url: https://invalid.url/spkenv/spk.git + extract: "v(.*)" + "#, + ), + ]); + + ws.repo + .list_package_versions(&"pkg-a".parse::().unwrap()) + .await + .expect_err("should fail when discovery is not possible"); +} + +#[rstest] +#[tokio::test] +async fn test_get_concrete_package_builds() { + let ws = TestWorkspace::new(&[ + ( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a + version: 1.0.0 + "#, + ), + ( + "pkg-b.spk.yaml", + r#" + pkg: pkg-b + version: 2.0.0 + "#, + ), + ]); + let ident = VersionIdent::new("pkg-a".parse().unwrap(), version!("1.0.0")); + let builds = ws.repo.get_concrete_package_builds(&ident).await.unwrap(); + assert_eq!(builds.len(), 1); + assert!(builds.contains(&ident.to_build_ident(Build::Source))); +} + +#[rstest] +#[tokio::test] +async fn test_read_package_from_storage() { + let ws = TestWorkspace::new(&[( + "pkg-a.spk.yaml", + r#" + template: + versions: + static: ["1.0.0"] + pkg: pkg-a/{{ version }} + "#, + )]); + let ident = "pkg-a/1.0.0/src".parse().unwrap(); + let build = ws.repo.read_package_from_storage(&ident).await.unwrap(); + assert_eq!(build.ident(), &ident); +} + +#[rstest] +#[tokio::test] +async fn test_read_package_from_storage_not_found() { + let ws = TestWorkspace::new(&[( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a + template: + versions: + static: ["1.0.0"] + "#, + )]); + let ident = "pkg-a/2.0.0/src".parse().unwrap(); + let err = ws.repo.read_package_from_storage(&ident).await.unwrap_err(); + assert!(matches!(err, crate::Error::PackageNotFound(_))); +} + +#[rstest] +#[tokio::test] +async fn test_read_package_from_storage_not_source() { + let ws = TestWorkspace::new(&[( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a + template: + versions: + static: ["1.0.0"] + "#, + )]); + let ident = VersionIdent::new("pkg-a".parse().unwrap(), version!("1.0.0")) + .to_build_ident(Build::BuildId(BuildId::default())); + let err = ws.repo.read_package_from_storage(&ident).await.unwrap_err(); + assert!(matches!(err, crate::Error::PackageNotFound(_))); +} + +#[rstest] +#[tokio::test] +async fn test_list_build_components() { + let ws = TestWorkspace::new(&[( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a + template: + versions: + static: ["1.0.0"] + "#, + )]); + let ident = "pkg-a/1.0.0/src".parse().unwrap(); + let components = ws.repo.list_build_components(&ident).await.unwrap(); + assert_eq!( + components, + vec![spk_schema::foundation::ident_component::Component::Source] + ); + + let ident = VersionIdent::new("pkg-a".parse().unwrap(), version!("1.0.0")) + .to_build_ident(Build::BuildId(BuildId::default())); + let err = ws.repo.list_build_components(&ident).await.unwrap_err(); + assert!(matches!(err, crate::Error::PackageNotFound(_))); +} + +#[rstest] +#[tokio::test] +async fn test_read_recipe() { + let ws = TestWorkspace::new(&[( + "pkg-a.spk.yaml", + r#" + pkg: pkg-a/{{ version }} + template: + versions: + static: ["1.0.0"] + "#, + )]); + let ident = "pkg-a/1.0.0".parse().unwrap(); + let recipe = ws.repo.read_recipe(&ident).await.unwrap(); + assert_eq!(recipe.ident(), &ident); + + let ident = "pkg-a/2.0.0".parse().unwrap(); + let err = ws.repo.read_recipe(&ident).await.unwrap_err(); + assert!(matches!(err, crate::Error::PackageNotFound(_))); +} diff --git a/crates/spk-workspace/Cargo.toml b/crates/spk-workspace/Cargo.toml index 85fb69feaf..05ffc916a6 100644 --- a/crates/spk-workspace/Cargo.toml +++ b/crates/spk-workspace/Cargo.toml @@ -12,9 +12,6 @@ description = { workspace = true } [lints] workspace = true -[features] -sentry = ["spk-solve/sentry"] - [dependencies] bracoxide = { workspace = true } dunce = { workspace = true } @@ -25,7 +22,6 @@ miette = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_yaml = { workspace = true } spk-schema = { workspace = true } -spk-solve = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } diff --git a/crates/spk-workspace/src/builder.rs b/crates/spk-workspace/src/builder.rs index 5c2cd01708..46491cda21 100644 --- a/crates/spk-workspace/src/builder.rs +++ b/crates/spk-workspace/src/builder.rs @@ -4,7 +4,8 @@ //! Find and/or build workspaces. -use std::collections::HashMap; +use std::collections::BTreeSet; +use std::error::Error; use crate::error; @@ -13,7 +14,8 @@ use crate::error; #[derive(Default)] pub struct WorkspaceBuilder { root: Option, - spec_files: HashMap, + spec_files: BTreeSet, + ignore_invalid_files: bool, } impl WorkspaceBuilder { @@ -68,10 +70,7 @@ impl WorkspaceBuilder { .unwrap_or(item.path.as_str()); let mut glob_results = glob::glob(pattern)?; while let Some(path) = glob_results.next().transpose()? { - self.spec_files - .entry(path) - .or_default() - .update(item.config.clone()); + self.spec_files.insert(path); } Ok(self) @@ -89,22 +88,35 @@ impl WorkspaceBuilder { ) -> Result { self.with_recipes_item(&crate::file::RecipesItem { path: glob::Pattern::new(pattern.as_ref())?, - config: Default::default(), }) } + /// When false, the workspace build will fail if any recipe files are invalid. + /// + /// This defaults to `false` because workspaces are used even in cases + /// where there is no explicit workspace file found. In these cases it + /// is seen as unexpected to fail on invalid files, but it can be enabled + /// in workspace-specific cases where the validation is appropriate. + pub fn with_ignore_invalid_files(mut self, ignore: bool) -> Self { + self.ignore_invalid_files = ignore; + self + } + /// Build the workspace as configured. pub fn build(self) -> Result { let mut workspace = super::Workspace::default(); - for (file, config) in self.spec_files { - match workspace.load_template_file_with_config(&file, config) { + for file in self.spec_files { + match workspace.load_template_file(&file) { Ok(_) => {} - Err(e) => { + Err(e) if self.ignore_invalid_files => { tracing::warn!( file = file.to_string_lossy().to_string(), - "ignoring template file: {e}" + err = %e, + cause = ?e.source().map(ToString::to_string), + "ignoring template file" ); } + Err(e) => return Err(e), } } Ok(workspace) diff --git a/crates/spk-workspace/src/file.rs b/crates/spk-workspace/src/file.rs index 21bc1ea5a0..7ceb6e52ff 100644 --- a/crates/spk-workspace/src/file.rs +++ b/crates/spk-workspace/src/file.rs @@ -2,15 +2,14 @@ // SPDX-License-Identifier: Apache-2.0 // https://github.com/spkenv/spk -use std::collections::BTreeSet; +//! Workspace file definition. +//! +//! The format and loading process for workspace yaml files. + use std::path::{Path, PathBuf}; -use std::str::FromStr; -use bracoxide::OxidizationError; -use bracoxide::tokenizer::TokenizationError; use serde::Deserialize; use spk_schema::foundation::FromYaml; -use spk_schema::version::Version; use crate::error::LoadWorkspaceFileError; @@ -75,11 +74,11 @@ impl WorkspaceFile { Err(LoadWorkspaceFileError::WorkspaceNotFound(cwd)) } } - +/// One item in the list of recipes for a workspace. #[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] pub struct RecipesItem { + /// The path to a recipe file or files in the workspace. pub path: glob::Pattern, - pub config: TemplateConfig, } impl<'de> serde::de::Deserialize<'de> for RecipesItem { @@ -101,10 +100,7 @@ impl<'de> serde::de::Deserialize<'de> for RecipesItem { E: serde::de::Error, { let path = glob::Pattern::new(v).map_err(serde::de::Error::custom)?; - Ok(RecipesItem { - path, - config: Default::default(), - }) + Ok(RecipesItem { path }) } fn visit_map(self, map: A) -> Result @@ -114,104 +110,14 @@ impl<'de> serde::de::Deserialize<'de> for RecipesItem { #[derive(Deserialize)] struct RawRecipeItem { path: String, - #[serde(flatten)] - config: TemplateConfig, } - let RawRecipeItem { path, config } = + let RawRecipeItem { path } = RawRecipeItem::deserialize(serde::de::value::MapAccessDeserializer::new(map))?; - let mut base = self.visit_str(&path)?; - base.config = config; - Ok(base) + self.visit_str(&path) } } deserializer.deserialize_any(RecipeCollectorVisitor) } } - -#[derive(Debug, Clone, Hash, PartialEq, Eq, Ord, PartialOrd, Default)] -pub struct TemplateConfig { - /// Ordered set of versions that this template can produce. - /// - /// An empty set of versions does not mean that no versions can - /// be produced, but rather that any can be attempted. It's also - /// typical for a template to have a single hard-coded version inside - /// and so not need to specify values for this field. - pub versions: BTreeSet, -} - -impl TemplateConfig { - /// Update this config with newly specified data. - /// - /// Default values in the provided `other` value do not - /// overwrite existing data in this instance. - pub fn update(&mut self, other: Self) { - let Self { versions } = other; - if !versions.is_empty() { - self.versions = versions; - } - } -} - -impl<'de> serde::de::Deserialize<'de> for TemplateConfig { - fn deserialize(deserializer: D) -> Result - where - D: serde::de::Deserializer<'de>, - { - struct TemplateConfigVisitor; - - impl<'de> serde::de::Visitor<'de> for TemplateConfigVisitor { - type Value = TemplateConfig; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("additional recipe collection configuration") - } - - fn visit_map(self, map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - #[derive(Deserialize)] - struct RawConfig { - versions: Vec, - } - - let raw_config = - RawConfig::deserialize(serde::de::value::MapAccessDeserializer::new(map))?; - let mut base = TemplateConfig::default(); - for (i, version_expr) in raw_config.versions.into_iter().enumerate() { - let expand_result = bracoxide::bracoxidize(&version_expr); - let expanded = match expand_result { - Ok(expanded) => expanded, - Err(OxidizationError::TokenizationError(TokenizationError::NoBraces)) - | Err(OxidizationError::TokenizationError( - TokenizationError::EmptyContent, - )) - | Err(OxidizationError::TokenizationError( - TokenizationError::FormatNotSupported, - )) => { - vec![version_expr] - } - Err(err) => { - return Err(serde::de::Error::custom(format!( - "invalid brace expansion in position {i}: {err:?}" - ))); - } - }; - for version in expanded { - let parsed = Version::from_str(&version).map_err(|err| { - serde::de::Error::custom(format!( - "brace expansion in position {i} produced invalid version '{version}': {err}" - )) - })?; - base.versions.insert(parsed); - } - } - Ok(base) - } - } - - deserializer.deserialize_map(TemplateConfigVisitor) - } -} diff --git a/crates/spk-workspace/src/lib.rs b/crates/spk-workspace/src/lib.rs index c8a038e114..6bd81ebd6e 100644 --- a/crates/spk-workspace/src/lib.rs +++ b/crates/spk-workspace/src/lib.rs @@ -7,11 +7,11 @@ //! The [`WorkspaceFile`] is used to load [`Workspace`] configurations from //! yaml files on disk. -#![deny(missing_docs)] +#![warn(missing_docs)] pub mod builder; pub mod error; -mod file; +pub mod file; mod workspace; pub use file::WorkspaceFile; diff --git a/crates/spk-workspace/src/workspace.rs b/crates/spk-workspace/src/workspace.rs index 0e620daf7c..484c5692ae 100644 --- a/crates/spk-workspace/src/workspace.rs +++ b/crates/spk-workspace/src/workspace.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::str::FromStr; use spk_schema::name::{PkgName, PkgNameBuf}; +use spk_schema::template::DiscoverVersions; use spk_schema::version_range::{LowestSpecifiedRange, Ranged}; use spk_schema::{SpecTemplate, Template, TemplateExt}; @@ -23,19 +24,14 @@ mod workspace_test; /// can be used to determine the number and order of /// packages to be built in order to efficiently satisfy /// and entire set of requirements for an environment. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Workspace { + root: Option, /// Spec templates available in this workspace. /// /// A workspace may contain multiple recipes for a single /// package. - pub(crate) templates: HashMap>, -} - -#[derive(Debug, Clone)] -pub struct ConfiguredTemplate { - pub template: SpecTemplate, - pub config: crate::file::TemplateConfig, + pub(crate) templates: HashMap>, } impl Workspace { @@ -44,8 +40,16 @@ impl Workspace { crate::builder::WorkspaceBuilder::default() } + /// The logical root directory for this workspace. + /// + /// May be none in cases where the workspace was constructed + /// manually or is the default. + pub fn root(&self) -> Option<&std::path::Path> { + self.root.as_deref() + } + /// Iterate over all templates in the workspace. - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.templates .iter() .flat_map(|(name, templates)| templates.iter().map(|t| (name.as_ref(), t))) @@ -80,7 +84,7 @@ impl Workspace { pub fn find_or_load_package_template( &mut self, package: S, - ) -> Result<&ConfiguredTemplate, FindOrLoadPackageTemplateError> + ) -> Result<&SpecTemplate, FindOrLoadPackageTemplateError> where S: AsRef, { @@ -119,6 +123,8 @@ impl Workspace { ident.name() ); self.find_package_template_for_version(ident.name(), range) + } else if let Ok(ident) = spk_schema::ident::RangeIdent::from_str(package) { + self.find_package_template_for_version(&ident.name, ident.version) } else { tracing::debug!("Find package template by path: {package}"); self.find_package_template_by_file(std::path::Path::new(package)) @@ -140,15 +146,12 @@ impl Workspace { &self, package: &PkgName, range: R, - ) -> Vec<&ConfiguredTemplate> { + ) -> Vec<&SpecTemplate> { self.find_package_templates(package) .into_iter() .filter(|t| { - t.config.versions.is_empty() - || t.config - .versions - .iter() - .any(|v| range.is_applicable(v).is_ok()) + t.discover_versions() + .is_ok_and(|v| v.iter().any(|v| range.is_applicable(v).is_ok())) }) .collect::>() } @@ -156,7 +159,7 @@ impl Workspace { /// Find a package templates for the requested package, if any. /// /// Either a package name or filename can be provided. - pub fn find_package_templates(&self, name: &PkgName) -> Vec<&ConfiguredTemplate> { + pub fn find_package_templates(&self, name: &PkgName) -> Vec<&SpecTemplate> { if let Some(templates) = self.templates.get(name) { templates.iter().collect() } else { @@ -164,11 +167,17 @@ impl Workspace { } } + /// Like [`Self::find_package_templates`], but returns mutable references. + pub fn find_package_templates_mut(&mut self, name: &PkgName) -> Vec<&mut SpecTemplate> { + if let Some(templates) = self.templates.get_mut(name) { + templates.iter_mut().collect() + } else { + Default::default() + } + } + /// Find package templates by their file path, if any. - pub fn find_package_template_by_file( - &self, - file: &std::path::Path, - ) -> Vec<&ConfiguredTemplate> { + pub fn find_package_template_by_file(&self, file: &std::path::Path) -> Vec<&SpecTemplate> { // Attempt to canonicalize `file` using the same function that the // workspace uses as it locates files, to have a chance of matching // one of the entries in the workspace by comparing to its @@ -177,7 +186,7 @@ impl Workspace { self.templates .values() .flat_map(|templates| templates.iter()) - .filter(|t| t.template.file_path() == file_path) + .filter(|t| t.file_path() == file_path) .collect() } @@ -188,19 +197,7 @@ impl Workspace { pub fn load_template_file>( &mut self, path: P, - ) -> Result<&mut ConfiguredTemplate, error::BuildError> { - self.load_template_file_with_config(path, Default::default()) - } - - /// Load an additional template into this workspace from an arbitrary path on disk. - /// - /// No checks are done to ensure that this template actually appears in or - /// logically belongs in this workspace. - pub fn load_template_file_with_config>( - &mut self, - path: P, - config: crate::file::TemplateConfig, - ) -> Result<&mut ConfiguredTemplate, error::BuildError> { + ) -> Result<&mut SpecTemplate, error::BuildError> { let template = spk_schema::SpecTemplate::from_file(path.as_ref()).map_err(|source| { error::BuildError::TemplateLoadError { file: path.as_ref().to_owned(), @@ -217,18 +214,9 @@ impl Workspace { file: path.as_ref().to_owned(), }); }; - let loaded_path = template.file_path(); let by_name = self.templates.entry(name.clone()).or_default(); - let existing = by_name - .iter() - .position(|t| t.template.file_path() == loaded_path); - if let Some(existing) = existing { - by_name[existing].config.update(config); - Ok(&mut by_name[existing]) - } else { - by_name.push(ConfiguredTemplate { template, config }); - Ok(by_name.last_mut().expect("just pushed something")) - } + by_name.push(template); + Ok(by_name.last_mut().expect("just pushed something")) } } @@ -253,7 +241,7 @@ pub enum FindOrLoadPackageTemplateError { /// The result of the [`Workspace::find_package_template`] function. pub type FindPackageTemplateResult<'workspace> = - Result<&'workspace ConfiguredTemplate, FindPackageTemplateError>; + Result<&'workspace SpecTemplate, FindPackageTemplateError>; /// Possible errors for [`Workspace::find_package_template`] and related functions. #[derive(Debug, thiserror::Error, miette::Diagnostic)] @@ -268,7 +256,7 @@ pub enum FindPackageTemplateError { /// files in the current repository. #[error("Multiple package specs in current workspace:\n{}", self.formatted_packages_list())] #[diagnostic(help = "ensure that you specify a package name, file path or version")] - MultipleTemplates(Vec), + MultipleTemplates(Vec), /// No package was specifically requested, and there no template /// files in the current repository. #[error("No package specs found in current workspace")] @@ -293,11 +281,14 @@ impl FindPackageTemplateError { // attempt to strip the current working directory from each path // because in most cases it was loaded from the active workspace // and the additional path prefix is just noise - let path = configured.template.file_path(); + let path = configured.file_path(); let path = path.strip_prefix(&here).unwrap_or(path).to_string_lossy(); let mut versions = configured - .config - .versions + .discover_versions() + .inspect_err(|err| { + tracing::warn!(?path, "encountered error discovering versions: {err}") + }) + .unwrap_or_default() .iter() .map(ToString::to_string) .collect::>() diff --git a/crates/spk-workspace/src/workspace_test.rs b/crates/spk-workspace/src/workspace_test.rs index 418b71fa7a..2f3637cb0f 100644 --- a/crates/spk-workspace/src/workspace_test.rs +++ b/crates/spk-workspace/src/workspace_test.rs @@ -6,6 +6,7 @@ use std::vec; use rstest::{fixture, rstest}; use spk_schema::Template; +use spk_schema::template::DiscoverVersions; use super::Workspace; @@ -45,13 +46,9 @@ fn test_config_specialization(tmpdir: tempfile::TempDir) { recipes: vec![ crate::file::RecipesItem { path: "*.spk.yaml".parse().unwrap(), - config: Default::default(), }, crate::file::RecipesItem { path: "pkg-a.spk.yaml".parse().unwrap(), - config: crate::file::TemplateConfig { - versions: vec![v1.clone()].into_iter().collect(), - }, }, ], }) @@ -61,9 +58,8 @@ fn test_config_specialization(tmpdir: tempfile::TempDir) { let found = workspace.find_package_template("pkg-a").unwrap(); assert!( - found.config.versions.contains(&v1), - "config specialization should apply based on workspace file order, got: {:?}", - found.config + found.discover_versions().unwrap().contains(&v1), + "config specialization should apply based on workspace file order, got: {found:?}", ) } @@ -131,7 +127,6 @@ fn test_workspace_find_template( let result = result.expect("should be found"); assert_eq!( result - .template .file_path() .file_name() .expect("template has file name") @@ -161,19 +156,12 @@ fn test_workspace_find_by_version(tmpdir: tempfile::TempDir) { recipes: vec![ crate::file::RecipesItem { path: "*.spk.yaml".parse().unwrap(), - config: Default::default(), }, crate::file::RecipesItem { path: "1.spk.yaml".parse().unwrap(), - config: crate::file::TemplateConfig { - versions: vec![v1.clone()].into_iter().collect(), - }, }, crate::file::RecipesItem { path: "2.spk.yaml".parse().unwrap(), - config: crate::file::TemplateConfig { - versions: vec!["2.0.0".parse().unwrap()].into_iter().collect(), - }, }, ], }) @@ -195,8 +183,7 @@ fn test_workspace_find_by_version(tmpdir: tempfile::TempDir) { .find_package_template("my-package/1") .expect("should find template when multiple exist but an unambiguous version is given"); assert!( - found.config.versions.contains(&v1), - "should select the requested version, got: {:?}", - found.config + found.discover_versions().unwrap().contains(&v1), + "should select the requested version, got: {found:?}", ) } diff --git a/crates/spk/Cargo.toml b/crates/spk/Cargo.toml index ac8255c135..f067548acf 100644 --- a/crates/spk/Cargo.toml +++ b/crates/spk/Cargo.toml @@ -50,6 +50,7 @@ cli = [ "dep:spk-cmd-render", "dep:spk-cmd-repo", "dep:spk-cmd-test", + "dep:spk-cmd-workspace", ] statsd = ["dep:statsd", "spk-solve/statsd"] @@ -80,6 +81,7 @@ spk-cmd-make-recipe = { workspace = true, optional = true } spk-cmd-render = { workspace = true, optional = true } spk-cmd-repo = { workspace = true, optional = true } spk-cmd-test = { workspace = true, optional = true } +spk-cmd-workspace = { workspace = true, optional = true } spk-exec = { workspace = true } spk-schema = { workspace = true } spk-solve = { workspace = true } diff --git a/crates/spk/src/cli.rs b/crates/spk/src/cli.rs index eb951d37d1..b831eb8de4 100644 --- a/crates/spk/src/cli.rs +++ b/crates/spk/src/cli.rs @@ -29,6 +29,7 @@ use spk_cmd_make_source::cmd_make_source; use spk_cmd_render::cmd_render; use spk_cmd_repo::cmd_repo; use spk_cmd_test::cmd_test; +use spk_cmd_workspace::cmd_workspace; use spk_schema::foundation::format::FormatError; #[cfg(feature = "statsd")] use spk_solve::{ @@ -175,6 +176,7 @@ pub enum Command { Undeprecate(cmd_undeprecate::Undeprecate), Version(cmd_version::Version), View(cmd_view::View), + Workspace(cmd_workspace::Workspace), } // At the time of writing, enum_dispatch is not working to generate this code @@ -215,6 +217,7 @@ impl Run for Command { Command::Undeprecate(cmd) => cmd.run().await, Command::Version(cmd) => cmd.run().await, Command::View(cmd) => cmd.run().await, + Command::Workspace(cmd) => cmd.run().await, } } } @@ -251,6 +254,7 @@ impl CommandArgs for Command { Command::Undeprecate(cmd) => cmd.get_positional_args(), Command::Version(cmd) => cmd.get_positional_args(), Command::View(cmd) => cmd.get_positional_args(), + Command::Workspace(cmd) => cmd.get_positional_args(), } } } diff --git a/docs/develop/structure.md b/docs/develop/structure.md index 1ac541dd01..1de4f58aac 100644 --- a/docs/develop/structure.md +++ b/docs/develop/structure.md @@ -25,6 +25,8 @@ generate_s ==> sp[Package] generate_b ==> bp[Package] ``` +The lifecycle begins with a `Template`, but only package spec file containing a top-level `template` block is treated as a template. The `render` step can use metadata within this block to discover and validate input `variables` (such as the version number) against external sources. Once validated, the metadata is stripped and the variables are injected into the rest of the file to produce a concrete, single-version `Recipe`. This recipe is then used to generate the source and binary packages. For more information, see the guide on [Templated Recipes]({{< ref "../use/create/templated-recipes" >}}). + ### Metadata vs Payloads Each package is made up of two pieces which are important to differentiate: the package `payload` and `specification (spec)`. The package payload is the set of files on disk that package 'contains'. When you install a package, the payload is the files that you actually see in `/spfs`. The package specification (or metadata) is information about the package: how it was built, what it's dependencies are, and everything else that's important for both spk and developers to know. diff --git a/docs/ref/api/v0/package.md b/docs/ref/api/v0/package.md index 688ac5bb2c..c8babe8877 100644 --- a/docs/ref/api/v0/package.md +++ b/docs/ref/api/v0/package.md @@ -10,16 +10,17 @@ This document details each data structure and field that does or can exist withi The root package spec defines which fields can and should exist at the top level of a spec file. -| Field | Type | Description | -| ---------- | --------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| pkg | _[Identifier](#identifier)_ | The name and version number of this package | -| meta | [Meta](#meta) | Extra package metadata such as description, license, etc | -| compat | _[Compat](#compat)_ | The compatibility semantics of this packages versioning scheme | -| deprecated | _boolean_ | True if this package has been deprecated, this is usually reserved for internal use only and should not generally be specified directly in spec files | -| sources | _List[[SourceSpec](#sourcespec)]_ | Specifies where to get source files for building this package | -| build | _[BuildSpec](#buildspec)_ | Specifies how the package is to be built | -| tests | _List[[TestSpec](#testspec)]_ | Specifies any number of tests to validate the package and software | -| install | _[InstallSpec](#installspec)_ | Specifies how the package is to be installed | +| Field | Type | Description | +| ---------- | --------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| pkg | _[Identifier](#identifier)_ | The name and version number of this package | +| template | _[TemplateSpec](#templatespec)_ | (Optional) If present, this file is treated as a recipe template. Contains metadata for discovering and rendering versions. See [Templated Recipes]({{< ref "../../use/create/templated-recipes" >}}) for details. | +| meta | [Meta](#meta) | Extra package metadata such as description, license, etc | +| compat | _[Compat](#compat)_ | The compatibility semantics of this packages versioning scheme | +| deprecated | _boolean_ | True if this package has been deprecated, this is usually reserved for internal use only and should not generally be specified directly in spec files | +| sources | _List[[SourceSpec](#sourcespec)]_ | Specifies where to get source files for building this package | +| build | _[BuildSpec](#buildspec)_ | Specifies how the package is to be built | +| tests | _List[[TestSpec](#testspec)]_ | Specifies any number of tests to validate the package and software | +| install | _[InstallSpec](#installspec)_ | Specifies how the package is to be installed | ## Meta @@ -468,4 +469,51 @@ The package identifier takes the form `[/[/]]`, where: ## Compat -Specifies the compatibility contract of a version number. The compat string is a dot-separated set of characters that define contract, for example `x.a.b` (the default contract) says that major version changes are not compatible, minor version changes provides **A**PI compatibility, and patch version changes provide **B**inary compatibility. +Specifies the compatibility contract of a version number. The compat string is a dot-separated set of characters that define contract, for example `x.a.b` (the default contract) says that major version changes are not compatible, minor version changes provides **A**PI compatibility, and patch version changes provide **B**inary compatibility. Multiple characters can be put together if necessary: `x.ab`. + +If not specified, the default value for this field is: `x.a.b`. This means that at build time and on the command line, when API compatibility is needed, any minor version of this package can be considered compatible (eg `my-package/1.0.0` could resolve any `my-package/1.*`). When resolving dependencies however, when binary compatibility is needed, only the patch version is considered (eg `my-package/1.0.0` could resolve any `my-package/1.0.*`). + +Pre-releases and post-releases of the same version are treated as compatible, however this can be controlled by adding an extra compatibility clause to the `compat` field. For example, `x.x.x-x+x` would mark a build as completely incompatible with any other build, including other pre- or post-releases of the same version. + +```yaml +pkg: my-package/1.0.0 +compat: x.a.b +# where major versions are not compatible +# minor versions are API-compatible +# patch versions are binary compatible +``` + +The compat field of the new version is checked before install/update. Because of this, the compat field is more af a contract with past versions rather than future ones. Although it's recommended that your version compatibility remain constant for all versions of a package, this is not strictly required. + +## TemplateSpec + +The `template` block identifies a spec file as a template for building multiple versions of a package. It contains metadata that the build system uses to discover buildable versions and render a concrete recipe for a specific version. This block is only used during the templating phase and is not part of the final, evaluated recipe. + +For a complete guide, see [Templated Recipes and Version Discovery]({{< ref "../../use/create/templated-recipes" >}}). + +| Field | Type | Description | +| -------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| for | _str_ | **Required**. The name of the package that this template builds (e.g., `python`). | +| versions | _[VersionDiscovery](#versiondiscovery)_ | **Required**. An object that defines how the list of buildable versions is generated, either statically or through discovery. | + +### VersionDiscovery + +| Field | Type | Description | +| -------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| discover | _[DiscoveryStrategy](#discoverystrategy)_ | An object defining the dynamic strategy for discovering versions from an external source. | + +### DiscoveryStrategy + +Currently, only one discovery strategy is supported. + +| Field | Type | Description | +| -------- | ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| git_tags | _[GitTagsDiscovery](#gittagsdiscovery)_ | The configuration for discovering versions from the tags of a git repository. | + +### GitTagsDiscovery + +| Field | Type | Description | +| ------- | ----- | ------------------------------------------------------------------------------------------------------- | +| url | _str_ | The URL of the git repository to query for tags. | +| match | _str_ | A glob-style pattern to filter which tags are considered (e.g., `v3.9.*`). | +| extract | _str_ | (Optional) A regular expression with one capture group to extract the version string from the tag name. | \ No newline at end of file diff --git a/docs/use/create/templated-recipes.md b/docs/use/create/templated-recipes.md new file mode 100644 index 0000000000..46fe43fabe --- /dev/null +++ b/docs/use/create/templated-recipes.md @@ -0,0 +1,75 @@ +--- +title: Templated Recipes & Version Discovery +summary: Create reusable package recipes that can build multiple software versions. +weight: 15 +--- + +For many packages, especially foundational ones like `python` or `gcc`, it is inefficient to create and maintain a separate spec file for every single patch release. SPK provides a powerful solution to this problem: **Templated Recipes**. + +A single templated recipe can define the build process for an entire family of versions (e.g., all Python 3.9 releases). It works by pairing a generic build script with a set of rules for discovering which versions are available from an upstream source, like a git repository. + +### Becoming a Template: The `template` Metadata Block + +A standard package spec file becomes a template by including a top-level `template` block. This block contains metadata that is used by the SPK templating engine. It is not part of the final, rendered recipe. + +Here is an example of a recipe for `python` that has been converted into a template: + +```yaml +# In packages/python/python3.spk.yaml + +# This 'template' block is metadata for the build system. +# It signifies that this file is a template, not a +# direct recipe. +template: + # Declares that this template is used for building the + # package named 'python'. + for: python + # Defines how to discover the supported versions. + versions: + discover: + # Strategy 1: Check a git repository's tags. + git_tags: + url: https://github.com/python/cpython.git + # A pattern to match against the tags. + match: v3.9.* + # An optional regex to extract the clean version number from the tag. + # This would turn "v3.9.22" into "3.9.22". + extract: 'v(.*)' + +# --- The actual recipe template begins below --- +# The build system will process the rest of this file using +# a jinja2 template engine, injecting the discovered version. +# The 'template' block above will be stripped before this stage. + +api: v0/package +pkg: python/{{ version }} +build: + # The build script can now use the {{ version }} variable, + # which will be populated by the templating engine. + script: + - ./configure --version={{ version }} + # ... etc +``` + +### The `template` Block In Detail + +- `for`: A **required** string that links the template to a package name. When a platform requests to build `python`, SPK knows to look for a template with `for: python`. +- `versions`: An object that defines how the list of buildable versions is generated. +- `versions.discover`: This key tells SPK to look for versions dynamically from an external source. +- `versions.discover.git_tags`: This specifies the "git tag" discovery strategy. + - `url`: The URL of the git repository to query. + - `match`: A glob-style pattern to filter the tags. + - `extract`: An optional regular expression with a capture group to extract the desired version string from the full tag name. If omitted, the full tag name is used. + +### The Workflow in Action + +This system makes adding new package versions effortless: + +1. A new version, `3.9.22`, is released, and a `v3.9.22` tag is pushed to the CPython git repository. **No changes are needed in your spk repository.** +2. A developer wants to use it. They simply request it in a platform or on the command line. +3. SPK finds the recipe template for `python`. +4. It runs the `discover` logic, querying the git repository for tags matching `v3.9.*`. It finds `v3.9.22`. +5. The request is now validated against this dynamically generated list of supported versions. +6. SPK then renders the template in-memory, replacing `{{ version }}` with `3.9.22`, and proceeds with the build. + +This "set it and forget it" approach removes the manual and error-prone step of updating a central allow-list of versions, and keeps the versioning logic tightly coupled with the recipe that uses it. diff --git a/docs/use/platforms.md b/docs/use/platforms.md index b1a5160f89..57c71215bb 100644 --- a/docs/use/platforms.md +++ b/docs/use/platforms.md @@ -91,3 +91,30 @@ requirements: atRuntime: 3.7 # - pkg: imath not present ``` + +### Interaction with Templated Recipes + +The true power of platforms is revealed when they are used in combination with [Templated Recipes]({{< ref "./create/templated-recipes" >}}). This pairing enables a highly automated and agile workflow for keeping your software platforms up-to-date. + +Imagine a new version of Python, `3.9.22`, is released. With a traditional setup, a developer would need to manually edit a central configuration file to "allow" this new version to be built. + +With platforms and templated recipes, the process is seamless: + +1. A developer simply updates a platform file to request the new version: + + ```yaml + # In my-dcc-platform.spk.yaml + requirements: + - pkg: python + build: + version: 3.9.22 # Updated + atBuild: =3.9.22 # Updated + atRuntime: 3.9 + ``` + +2. When this platform is built, SPK sees the build for `python/3.9.22`. +3. It finds the **templated recipe** for `python`. +4. It automatically runs the version discovery logic defined in the template (e.g., checking for new git tags). +5. It finds that `3.9.22` is a valid new version, renders a concrete recipe for it, and builds it on-demand. + +There is no second step. The platform definition itself becomes the single source of truth for driving software updates, and the system automatically builds what is necessary to satisfy it. This removes significant manual effort and allows your studio to adopt new software versions with much greater speed and reliability. diff --git a/packages/bzip2/bzip2.spk.yaml b/packages/bzip2/bzip2.spk.yaml index 34fdb95158..8ecfd0fee2 100644 --- a/packages/bzip2/bzip2.spk.yaml +++ b/packages/bzip2/bzip2.spk.yaml @@ -1,8 +1,15 @@ -pkg: bzip2/1.0.6 +template: + versions: + discovery: + gitTags: .* + url: https://github.com/libarchive/bzip2 + extract: bzip2-(\d+\.\d+\.\d+) + +pkg: bzip2/{{ version }} api: v0/package sources: -- tar: https://www.sourceware.org/pub/bzip2/bzip2-1.0.6.tar.gz +- tar: https://www.sourceware.org/pub/bzip2/bzip2-{{ version }}.tar.gz build: options: @@ -12,7 +19,7 @@ build: - pkg: gcc - pkg: stdfs script: - - cd bzip2-1.0.6 + - cd bzip2-{{ version }} - make install PREFIX=/spfs CFLAGS=-fPIC install: diff --git a/packages/cmake/cmake.spk.yaml b/packages/cmake/cmake.spk.yaml index 45e0eadab0..34d9f71249 100644 --- a/packages/cmake/cmake.spk.yaml +++ b/packages/cmake/cmake.spk.yaml @@ -1,5 +1,11 @@ -# {% set opt = opt | default_opts(version="3.18.2") %} -pkg: cmake/{{ opt.version }} +template: + versions: + discover: + gitTags: v3\.\d+\.\d+ + url: https://github.com/Kitware/CMake + extract: v(\d+\.\d+\.\d+) + +pkg: cmake/{{ version }} api: v0/package sources: @@ -8,20 +14,20 @@ sources: # the tar file. - path: ./ - script: - - export TARFILE=cmake-{{ opt.version }}-Linux-x86_64.tar.gz - - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v{{ opt.version }}/$TARFILE ; fi + - export TARFILE=cmake-{{ version }}-Linux-x86_64.tar.gz + - if [ ! -e ./$TARFILE ] ; then wget https://github.com/Kitware/CMake/releases/download/v{{ version }}/$TARFILE ; fi build: options: - - var: arch - - var: os + - var: arch + - var: os script: - mkdir -p build; cd build - tar -xvf - ../cmake-{{ opt.version }}-Linux-x86_64.tar.gz - --strip-components=1 - --exclude=doc - --exclude=Help + ../cmake-{{ version }}-Linux-x86_64.tar.gz + --strip-components=1 + --exclude=doc + --exclude=Help - rsync -rv ./ $PREFIX/ install: diff --git a/packages/gnu/gcc/gcc48.spk.yaml b/packages/gnu/gcc/gcc48.spk.yaml index 6e0e4bcb73..7eb2545c41 100644 --- a/packages/gnu/gcc/gcc48.spk.yaml +++ b/packages/gnu/gcc/gcc48.spk.yaml @@ -1,9 +1,12 @@ -# {% set opt = opt | default_opts(version="4.8.5") %} -pkg: gcc/{{ opt.version }} +template: + versions: + static: 4.8.5 + +pkg: gcc/{{ version }} api: v0/package sources: - - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-{{ opt.version }}/gcc-{{ opt.version }}.tar.gz + - tar: http://ftpmirror.gnu.org/gnu/gcc/gcc-{{ version }}/gcc-{{ version }}.tar.gz - path: patch-gcc46-texi.diff build: @@ -25,10 +28,10 @@ build: - pkg: coreutils - pkg: binutils - pkg: make - - pkg: gcc/<={{ opt.version }} + - pkg: gcc/<={{ version }} script: - - patch -d gcc-{{ opt.version }} -p0 /spfs/bin/python - - echo 'exec python2 "$@"' >> /spfs/bin/python - - chmod +x /spfs/bin/python - - ./configure - --prefix=${PREFIX} - CC=$CC - CXX=$CXX - LDFLAGS='-Wl,--rpath=/spfs/lib,-L/spfs/lib' - PKG_CONFIG_PATH=/spfs/share/pkgconfig:/spfs/lib/pkgconfig - CPPFLAGS='-I/spfs/include/ncurses' - --with-ensurepip=no - --enable-shared - "$UNICODE" - $DEBUG - - make -j$(nproc) - - make install - # remove test files that are just bloat - - find /spfs/lib/python* -name "test" -type d | xargs -r rm -rv - - find /spfs/lib/python* -name "*_test" -type d | xargs -r rm -rv - - ln -sf python2 /spfs/bin/python - # do not package pyc files, spfs is best when pyc files are not generated at all - - find /spfs -type f -name "*.pyc" | xargs rm - -tests: - - stage: install - script: - # Verify we built a python with the requested ABI - - python_abi=$(/spfs/bin/python -c 'import wheel.bdist_wheel; - print(wheel.bdist_wheel.get_abi_tag())') - - | - if [ "$python_abi" != "$SPK_OPT_abi" ]; then - echo "Python binary ABI does not match spk options: $python_abi != $SPK_OPT_abi" - exit 1 - fi - - stage: install - script: - # Verify bz2 support is available by importing and not getting a traceback - - test -z "$(/spfs/bin/python -c 'import bz2' 2>&1)" - -install: - environment: - - set: PYTHONDONTWRITEBYTECODE - value: 1 - requirements: - - pkg: gcc - fromBuildEnv: x.x - include: IfAlreadyPresent - - pkg: stdfs - components: - - name: run - files: - - /etc/ - - /bin/ - - /lib/ - - '!/lib/pkgconfig' - - name: build - uses: [run] - files: - - /include/ - - /lib/pkgconfig - - name: man - files: - - /share/man diff --git a/packages/python/python3.spk.yaml b/packages/python/python3.spk.yaml index 815708ca4b..8e182dfc56 100644 --- a/packages/python/python3.spk.yaml +++ b/packages/python/python3.spk.yaml @@ -1,16 +1,18 @@ -# {% set opt = opt | default_opts(version="3.7.3") %} -# {% set cpXX = opt.version | replace_regex(from="(\d+)\.(\d+).*", to="cp$1$2") %} -pkg: python/{{ opt.version }} +template: + versions: + discover: + gitTags: v3\.\d+\.\d+ + url: https://github.com/python/cpython + extract: v(\d+\.\d+\.\d+) + +# {% set cpXX = version | replace_regex(from="(\d+)\.(\d+).*", to="cp$1$2") %} +pkg: python/{{ version }} api: v0/package sources: - git: https://github.com/python/cpython - ref: v{{ opt.version }} + ref: v{{ version }} build: options: - - var: os - - var: arch - - var: centos - - pkg: gcc - pkg: stdfs - pkg: bzip2 - pkg: ncurses @@ -26,8 +28,6 @@ build: - var: debug default: off choices: [on, off] - variants: - - { gcc: 6.3, abi: "{{cpXX}}m", debug: off } script: - | case "$SPK_OPT_debug" in @@ -91,8 +91,6 @@ install: - set: PYTHONDONTWRITEBYTECODE value: 1 requirements: - - pkg: binutils - fromBuildEnv: Binary - pkg: gcc fromBuildEnv: Binary include: IfAlreadyPresent @@ -111,7 +109,7 @@ install: - /etc/ - /bin/ - /lib/ - - '!/lib/pkgconfig' + - "!/lib/pkgconfig" - name: build uses: [run] files: diff --git a/packages/zlib/zlib.spk.yaml b/packages/zlib/zlib.spk.yaml index 4ff94951dd..f7b89d021d 100644 --- a/packages/zlib/zlib.spk.yaml +++ b/packages/zlib/zlib.spk.yaml @@ -1,35 +1,36 @@ -pkg: zlib/1.2.11+r.1 +template: + versions: + discover: + gitTags: v1\.\d+\.\d+ + url: https://github.com/madler/zlib + extract: v(\d+\.\d+\.?\d*) + +pkg: zlib/{{ version }} api: v0/package - # - name: "zlib" - # - description: "Compression library" - # - license: Zlib - # - url: https://www.zlib.net +meta: + description: "Compression library" + license: Zlib + homepage: https://www.zlib.net sources: # This idiom can work with any of (a) a local clone, (b) a git submodule, # or (c) nothing (does a fresh clone). - path: ./ - script: - - if [ ! -d zlib ] ; then git clone https://github.com/madler/zlib.git -b v1.2.11 ; fi + - if [ ! -d zlib ] ; then git clone https://github.com/madler/zlib.git -b v{{ version }} ; fi build: options: - - pkg: stdfs # provides the default filesystem structure (bin, lib, etc) - - var: arch # rebuild if the arch changes - - var: os # rebuild if the os changes - - var: centos # rebuild if centos version changes - - pkg: gcc/6.3 + - pkg: stdfs + - pkg: gcc - pkg: cmake/^3.13 # Because this is a pure C library, just build with any gcc and don't # specify multiple gcc variants. script: - cmake -S zlib -B build - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_PREFIX=${PREFIX} + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${PREFIX} - cmake --build build --target install - variants: - - {gcc: 4.8} - - {gcc: 6.3} install: requirements: diff --git a/platforms/vfx-reference-2024.spk.yaml b/platforms/vfx-reference-2024.spk.yaml new file mode 100644 index 0000000000..8aa09d26d0 --- /dev/null +++ b/platforms/vfx-reference-2024.spk.yaml @@ -0,0 +1,6 @@ +api: v1/platform +platform: vfx-reference/2024 +requirements: + - pkg: python + build: + version: 3.10.0 diff --git a/workspace.spk.yaml b/workspace.spk.yaml index 46bfe12718..e8d6d7e551 100644 --- a/workspace.spk.yaml +++ b/workspace.spk.yaml @@ -1,28 +1,7 @@ api: v0/workspace recipes: - # collect all of the recipes in the workspace - packages/**/*.spk.yaml - - # some recipes require additional information - # which can be augmented even if they were already - # collected above - - - path: packages/python/python2.spk.yaml - # here, we define the specific versions that can - # be build from a recipe - versions: [2.7.18] - - - path: packages/python/python3.spk.yaml - # we can use bash-style brace expansion to define - # ranges of versions that are supported - versions: - - '3.7.{0..17}' - - '3.8.{0..20}' - - '3.9.{0..21}' - - '3.10.{0..16}' - - '3.11.{0..11}' - - '3.12.{0..8}' - - '3.13.{0..1}' + - platforms/**/*.spk.yaml diff --git a/workspace.spk.yml b/workspace.spk.yml deleted file mode 100644 index cc340e4146..0000000000 --- a/workspace.spk.yml +++ /dev/null @@ -1,4 +0,0 @@ -api: v0/workspace - -recipes: - - packages/**.spk.yml