Indexes 8: Fixes the spk build or mkb crash with indexes enabled

.github/workflows/coverage.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install prerequisites
         run: |
           apt-get update
-          apt-get install --yes build-essential libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git
+          apt-get install --yes build-essential libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git libcurl4-openssl-dev
           # spfs-fuse requires this option enabled
           echo user_allow_other >> /etc/fuse.conf
           FB_REL=https://github.com/google/flatbuffers/releases/

.github/workflows/rust.yml

@@ -122,7 +122,7 @@ jobs:
       - name: Prepare Container
         run: |
           apt-get update
-          apt-get install -y libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git
+          apt-get install -y build-essential libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git libcurl4-openssl-dev
           # spfs-fuse requires this option enabled
           echo user_allow_other >> /etc/fuse.conf
           FB_REL=https://github.com/google/flatbuffers/releases/
@@ -234,7 +234,7 @@ jobs:
       - name: Prepare Container
         run: |
           apt-get update
-          apt-get install -y build-essential libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git
+          apt-get install -y build-essential libcap2-bin sudo cmake tcsh rsync protobuf-compiler fuse libfuse-dev curl unzip pkg-config libssl-dev git libcurl4-openssl-dev
           # spfs-fuse requires this option enabled
           echo user_allow_other >> /etc/fuse.conf
           FB_REL=https://github.com/google/flatbuffers/releases/
@@ -244,7 +244,7 @@ jobs:
       - name: Setup cross-compilation
         if: matrix.platform == 'windows'
         run: |
-          apt-get install -y gcc-mingw-w64-x86-64
+          apt-get install -y gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64-win32
           rustup target add ${{ matrix.target }}
       - name: Configure cargo
         run: |

Cargo.toml

@@ -95,6 +95,7 @@ progress_bar_derive_macro = { path = "crates/progress_bar_derive_macro" }
 proptest = "1.0.0"
 prost = "0.13"
 rand = "0.8.5"
+rdkafka = { version = "0.39.0", features = ["cmake-build"] }
 regex = "1.6"
 relative-path = "1.3"
 resolvo = "0.9.1"

crates/spk-config/src/config.rs

@@ -204,6 +204,30 @@ pub struct HostOptions {
     pub distro_rules: HashMap<String, DistroRule>,
 }
 
+/// Helper for testing default kafka timeout ms when not specified in
+/// config
+fn default_kafka_message_timeout_ms() -> u64 {
+    5000
+}
+
+/// Configuration for using Kafka as a message channel.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct KafkaChannel {
+    /// List of brokers to connect to, specified as "hostname:port"
+    pub brokers: Vec<String>,
+    /// Topic name to push package updates to
+    pub package_updates_topic_name: Option<String>,
+    /// Message timeout in milliseconds, defaults to 5000 ms (5 seconds)
+    #[serde(default = "default_kafka_message_timeout_ms")]
+    pub message_timeout_ms: u64,
+}
+
+/// Types of message channels.
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub enum MessageChannel {
+    Kafka(KafkaChannel),
+}
+
 /// Configuration values for spk.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 #[serde(default)]
@@ -218,6 +242,7 @@ pub struct Config {
     pub metadata: Metadata,
     pub cli: Cli,
     pub host_options: HostOptions,
+    pub messaging: Vec<MessageChannel>,
 }
 
 impl Config {

crates/spk-schema/src/v0/indexed_package.rs

@@ -30,6 +30,7 @@ use crate::foundation::name::PkgName;
 use crate::foundation::spec_ops::prelude::*;
 use crate::foundation::version::{Compat, Compatibility, Version};
 use crate::ident::{Satisfy, VarRequest};
+use crate::option::VarOpt;
 use crate::package::OptionValues;
 use crate::spec::SpecTest;
 use crate::v0::EmbeddedPackageSpec;
@@ -156,6 +157,47 @@ impl IndexedPackage {
             <spk_proto::BuildIndex as flatbuffers::Follow>::follow(&self.buf[..], self.offset)
         }
     }
+
+    // Helper function for gathering and filtering downstream
+    // requirements trait implementation.
+    fn downstream_requirements<F>(&self, filter: F) -> Cow<'_, RequirementsList<RequestWithOptions>>
+    where
+        F: FnMut(&VarOpt) -> bool,
+    {
+        // This is a version of downstream_requirements() from
+        // v0/package_spec.rs modified for this kind of flatbuffer
+        // backed package.
+        let build_options = self.build_options();
+        let embedded = self.embedded();
+
+        let requests = build_options
+            .iter()
+            .filter_map(|opt| match opt {
+                Opt::Var(v) => Some(v.with_default_namespace(self.name())),
+                Opt::Pkg(_) => None,
+            })
+            .chain(embedded.iter().flat_map(|embed| {
+                embed.build().options.iter().filter_map(|opt| match opt {
+                    Opt::Var(v) => Some(v.with_default_namespace(embed.name())),
+                    Opt::Pkg(_) => None,
+                })
+            }))
+            .filter(filter)
+            .map(|o| {
+                VarRequest {
+                    // we are assuming that the var here will have a value because
+                    // this is a built binary package
+                    value: o.get_value(None).unwrap_or_default().into(),
+                    var: o.var,
+                    // Index doesn't store the an option's description
+                    description: None,
+                }
+            })
+            .map(RequestWithOptions::Var);
+        RequirementsList::<RequestWithOptions>::try_from_iter(requests)
+            .map(Cow::Owned)
+            .expect("build opts (from a RepoIndex) do not contain duplicates")
+    }
 }
 
 impl BuildOptions for IndexedPackage {
@@ -489,15 +531,9 @@ impl DownstreamRequirements for IndexedPackage {
         &self,
         _components: impl IntoIterator<Item = &'a Component>,
     ) -> Cow<'_, RequirementsList<RequestWithOptions>> {
-        // This is for build var requirements and inheritance used in
-        // building. This kinds of package has no build data stored.
-        let err = Error::SpkIndexedPackageDoesNotImplement(
-            "DownstreamRequirements".to_string(),
-            "downstream_build_requirements".to_string(),
-        );
-        // TODO: should this change the return value, update all the
-        // caller's handling, and return an error for this implementation?
-        unreachable!("{err}");
+        // This is used when doing a binary build to get additional
+        // information about the build environment.
+        self.downstream_requirements(|o| o.inheritance() != Inheritance::Weak)
     }
 
     fn downstream_runtime_requirements<'a>(
@@ -506,40 +542,7 @@ impl DownstreamRequirements for IndexedPackage {
     ) -> Cow<'_, RequirementsList<RequestWithOptions>> {
         // This is used when deprecating an embedded stub/package
         // and this is exercised during the automated repository tests.
-
-        // This is a version of downstream_runtime_requirements() and
-        // downstream_requirements() from v0/package_spec.rs modified
-        // for this kind of flatbuffer backed package.
-        let build_options = self.build_options();
-        let embedded = self.embedded();
-
-        let requests = build_options
-            .iter()
-            .filter_map(|opt| match opt {
-                Opt::Var(v) => Some(v.with_default_namespace(self.name())),
-                Opt::Pkg(_) => None,
-            })
-            .chain(embedded.iter().flat_map(|embed| {
-                embed.build().options.iter().filter_map(|opt| match opt {
-                    Opt::Var(v) => Some(v.with_default_namespace(embed.name())),
-                    Opt::Pkg(_) => None,
-                })
-            }))
-            .filter(|o| o.inheritance() == Inheritance::Strong || o.required)
-            .map(|o| {
-                VarRequest {
-                    // we are assuming that the var here will have a value because
-                    // this is a built binary package
-                    value: o.get_value(None).unwrap_or_default().into(),
-                    var: o.var,
-                    // Index doesn't store the an option's description
-                    description: None,
-                }
-            })
-            .map(RequestWithOptions::Var);
-        RequirementsList::<RequestWithOptions>::try_from_iter(requests)
-            .map(Cow::Owned)
-            .expect("build opts (from a RepoIndex) do not contain duplicates")
+        self.downstream_requirements(|o| o.inheritance() == Inheritance::Strong || o.required)
     }
 }
 

crates/spk-storage/Cargo.toml

@@ -44,6 +44,7 @@ nom = { workspace = true }
 nom-supreme = { workspace = true }
 once_cell = { workspace = true }
 paste = { workspace = true }
+rdkafka = { workspace = true }
 regex = { workspace = true }
 relative-path = { workspace = true }
 ring = { workspace = true }
@@ -57,6 +58,7 @@ spfs = { workspace = true }
 spk-config = { workspace = true }
 spk-proto = { workspace = true }
 spk-schema = { workspace = true }
+strum = { workspace = true }
 sys-info = "0.9.0"
 tar = "0.4.45"
 tempfile = { workspace = true }

crates/spk-storage/src/storage/flatbuffer_index_test.rs

@@ -22,6 +22,7 @@ use spk_schema::{
     ComponentSpec,
     Components,
     Deprecate,
+    DownstreamRequirements,
     OptionMap,
     OptionValues,
     Package,
@@ -262,7 +263,17 @@ fn assert_packages_are_equivalent(build_from_repo: Arc<Spec>, build_from_index:
 
     //  fn get_all_tests(&self) -> Vec<SpecTest> - not implemented by SolverPackageSpec
 
-    // DownstreamRequirements - not implemented by SolverPackageSpec
+    // DownstreamRequirements
+    assert_eq!(
+        build_from_repo.downstream_build_requirements([]),
+        build_from_index.downstream_build_requirements([]),
+        "downstream_build_requirements() don't match [{pkg}]",
+    );
+    assert_eq!(
+        build_from_repo.downstream_runtime_requirements([]),
+        build_from_index.downstream_runtime_requirements([]),
+        "downstream_runtime_requirements() don't match [{pkg}]",
+    );
 
     // OptionValues
     assert_eq!(