[1/4] feat(snapshot): template build-context archive store (posixfs + oss)

src/snapshot/image_export/service.rs

@@ -67,6 +67,7 @@ impl SnapshotImageService {
                     let repository = Arc::new(posixfs::PosixFsSnapshotRepository::new(
                         Arc::new(posixfs::PosixFsCatalogStore::new(root.clone())),
                         Arc::new(posixfs::PosixFsArtifactStore::new(root.clone())),
+                        posixfs::PosixFsTemplateBuildFileStore::new(&root),
                     ));
                     (repository, ManagedLayerLocator::PosixFs { root })
                 }
@@ -447,6 +448,7 @@ mod tests {
         let repository = posixfs::PosixFsSnapshotRepository::new(
             Arc::new(posixfs::PosixFsCatalogStore::new(root.clone())),
             Arc::new(posixfs::PosixFsArtifactStore::new(root.clone())),
+            posixfs::PosixFsTemplateBuildFileStore::new(&root),
         );
         let uncommitted =
             SnapshotRecord::template_waiting(SnapshotId::generate(), None, Default::default());

src/snapshot/manager.rs

@@ -84,6 +84,14 @@ impl SnapshotManager {
         self.repository.create(record).await
     }
 
+    /// Returns the shared build-context archive store, when the configured
+    /// repository backend provides one.
+    pub fn template_build_files(
+        &self,
+    ) -> Option<Arc<dyn crate::snapshot::repository::TemplateBuildFileStore>> {
+        self.repository.template_build_files()
+    }
+
     #[tracing::instrument(skip(self, metadata, manifest), fields(snapshot_id = %metadata.id))]
     pub async fn publish(
         &self,

src/snapshot/repository/backends/oss/build_files.rs

@@ -0,0 +1,167 @@
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use super::client::OssClient;
+use crate::snapshot::repository::build_files::{
+    generate_upload_token, is_valid_build_files_hash, is_valid_upload_token,
+    TemplateBuildFileStore, TemplateBuildUploadGrant,
+};
+use crate::snapshot::repository::{RepositoryError, RepositoryResult};
+
+const BUILD_FILES_PREFIX: &str = "template-build-files";
+
+/// Build-context archive store backed by the OSS repository bucket.
+///
+/// Layout: `template-build-files/{hash}.tar` plus durable bearer grants under
+/// `template-build-files/upload-grants/`. Retention is delegated to bucket
+/// lifecycle rules; archives are cache entries the SDK re-uploads when absent.
+pub(crate) struct OssTemplateBuildFileStore {
+    client: Arc<OssClient>,
+}
+
+impl OssTemplateBuildFileStore {
+    pub(crate) fn new(client: Arc<OssClient>) -> Arc<Self> {
+        Arc::new(Self { client })
+    }
+
+    fn archive_key(hash: &str) -> RepositoryResult<String> {
+        if !is_valid_build_files_hash(hash) {
+            return Err(RepositoryError::InvalidRequest {
+                reason: format!("invalid build files hash '{hash}'"),
+            });
+        }
+        Ok(format!("{BUILD_FILES_PREFIX}/{hash}.tar"))
+    }
+
+    fn grant_key(token: &str) -> Option<String> {
+        is_valid_upload_token(token)
+            .then(|| format!("{BUILD_FILES_PREFIX}/upload-grants/{token}.json"))
+    }
+
+    /// Reads a grant record, mapping an absent object to `None`.
+    async fn read_grant(&self, key: &str) -> RepositoryResult<Option<TemplateBuildUploadGrant>> {
+        let bytes = match self.client.get_bytes(key).await {
+            Ok(bytes) => bytes,
+            Err(error) if OssClient::is_not_found_error(&error) => return Ok(None),
+            Err(error) => return Err(RepositoryError::backend("read upload grant", error)),
+        };
+        serde_json::from_slice(&bytes)
+            .map(Some)
+            .map_err(|error| RepositoryError::backend("parse upload grant", error))
+    }
+}
+
+#[async_trait]
+impl TemplateBuildFileStore for OssTemplateBuildFileStore {
+    async fn exists(&self, hash: &str) -> RepositoryResult<bool> {
+        let key = Self::archive_key(hash)?;
+        self.client
+            .exists(&key)
+            .await
+            .map_err(|error| RepositoryError::backend("check build archive", error))
+    }
+
+    async fn import(&self, hash: &str, staged: &Path) -> RepositoryResult<()> {
+        let key = Self::archive_key(hash)?;
+        // This is a bandwidth-saving fast path, not an atomic check-and-create:
+        // concurrent uploads may both complete. Each object publication is
+        // atomic, and the upload protocol treats repeated uploads for one hash
+        // as equivalent build input.
+        if self
+            .client
+            .exists(&key)
+            .await
+            .map_err(|error| RepositoryError::backend("check build archive", error))?
+        {
+            return Ok(());
+        }
+        self.client
+            .put_file(&key, staged)
+            .await
+            .map_err(|error| RepositoryError::backend("upload build archive", error))
+    }
+
+    async fn materialize(
+        &self,
+        hash: &str,
+        scratch_dir: &Path,
+    ) -> RepositoryResult<Option<PathBuf>> {
+        let key = Self::archive_key(hash)?;
+        let dest = scratch_dir.join(format!("{hash}.tar"));
+        match self.client.get_to_file(&key, &dest).await {
+            Ok(_) => Ok(Some(dest)),
+            Err(error) if OssClient::is_not_found_error(&error) => Ok(None),
+            Err(error) => Err(RepositoryError::backend("download build archive", error)),
+        }
+    }
+
+    async fn create_upload_grant(
+        &self,
+        template_id: &str,
+        hash: &str,
+        expires_unix: i64,
+    ) -> RepositoryResult<String> {
+        let token = generate_upload_token();
+        let key = Self::grant_key(&token).expect("generated token is valid");
+        let grant = serde_json::to_vec(&TemplateBuildUploadGrant::new(
+            template_id,
+            hash,
+            expires_unix,
+        ))
+        .map_err(|error| RepositoryError::backend("serialize upload grant", error))?;
+        self.client
+            .put_bytes(&key, grant)
+            .await
+            .map_err(|error| RepositoryError::backend("write upload grant", error))?;
+        Ok(token)
+    }
+
+    async fn verify_upload_grant(
+        &self,
+        token: &str,
+        template_id: &str,
+        hash: &str,
+        expires_unix: i64,
+        now_unix: i64,
+    ) -> RepositoryResult<bool> {
+        let Some(key) = Self::grant_key(token) else {
+            return Ok(false);
+        };
+        // Deliberately does not delete the object: verification must leave the
+        // upload URL usable for a retry.
+        let Some(grant) = self.read_grant(&key).await? else {
+            return Ok(false);
+        };
+        Ok(grant.authorizes(template_id, hash, expires_unix, now_unix))
+    }
+
+    async fn claim_upload_grant(
+        &self,
+        token: &str,
+        template_id: &str,
+        hash: &str,
+        expires_unix: i64,
+        now_unix: i64,
+    ) -> RepositoryResult<bool> {
+        let Some(key) = Self::grant_key(token) else {
+            return Ok(false);
+        };
+        let Some(grant) = self.read_grant(&key).await? else {
+            return Ok(false);
+        };
+        if !grant.authorizes(template_id, hash, expires_unix, now_unix) {
+            return Ok(false);
+        }
+        // Consume the grant so the upload URL cannot normally be replayed.
+        // S3-compatible stores offer no conditional delete, so simultaneous
+        // replays of one token can both observe the grant and publish the same
+        // logical build input.
+        self.client
+            .delete(&key)
+            .await
+            .map_err(|error| RepositoryError::backend("consume upload grant", error))?;
+        Ok(true)
+    }
+}

src/snapshot/repository/backends/oss/mod.rs

@@ -1,3 +1,4 @@
+mod build_files;
 mod client;
 mod config;
 mod layout;

src/snapshot/repository/backends/oss/repository.rs

@@ -42,6 +42,7 @@ pub(crate) struct OssSnapshotRepository {
     client: Arc<OssClient>,
     snapshot_image_storage: SnapshotImageStoragePolicy,
     acr_exporter: AcrDiskImageExporter,
+    build_files: Arc<super::build_files::OssTemplateBuildFileStore>,
 }
 
 const MAX_ALIAS_BIND_ATTEMPTS: usize = 5;
@@ -51,10 +52,12 @@ impl OssSnapshotRepository {
         client: Arc<OssClient>,
         snapshot_image_storage: SnapshotImageStoragePolicy,
     ) -> Self {
+        let build_files = super::build_files::OssTemplateBuildFileStore::new(Arc::clone(&client));
         Self {
             client,
             snapshot_image_storage,
             acr_exporter: AcrDiskImageExporter::new(),
+            build_files,
         }
     }
 
@@ -148,6 +151,15 @@ fn fallback_to_object_storage_would_mix_sources(
 
 #[async_trait]
 impl SnapshotRepository for OssSnapshotRepository {
+    fn template_build_files(
+        &self,
+    ) -> Option<Arc<dyn crate::snapshot::repository::TemplateBuildFileStore>> {
+        Some(Arc::clone(&self.build_files)
+            as Arc<
+                dyn crate::snapshot::repository::TemplateBuildFileStore,
+            >)
+    }
+
     async fn create(&self, record: SnapshotRecord) -> RepositoryResult<SnapshotRecord> {
         if !matches!(record.source, SnapshotSource::Template { .. }) {
             return Err(RepositoryError::InvalidRequest {

src/snapshot/repository/backends/posixfs/backend.rs

@@ -7,11 +7,13 @@ use tokio::task;
 
 use super::super::shared_runtime_cache_root;
 use super::artifacts::{CollectedBuiltArtifacts, PosixFsArtifactStore};
+use super::build_files::PosixFsTemplateBuildFileStore;
 use super::catalog::PosixFsCatalogStore;
 use super::runtime::PosixFsRuntimeResolver;
 use crate::image::cache::{local_image_services_from_global_config, OverlaybdLayerStore};
 use crate::sandbox::FirecrackerSnapshotManifest;
 use crate::snapshot::artifact_cache::LocalArtifactCache;
+use crate::snapshot::repository::build_files::TemplateBuildFileStore;
 use crate::snapshot::repository::interfaces::{SnapshotRepository, SnapshotRuntimeResolver};
 use crate::snapshot::repository::{RepositoryError, RepositoryResult, SnapshotListFilter};
 use crate::snapshot::types::{
@@ -72,9 +74,11 @@ impl PosixFsBackend {
         let runtime_cache_root = runtime_cache_root.unwrap_or_else(|| cache_root.join("runtime"));
         let catalog_store = Arc::new(PosixFsCatalogStore::new(root.clone()));
         let artifact_store = Arc::new(PosixFsArtifactStore::new(root.clone()));
+        let build_files = PosixFsTemplateBuildFileStore::new(&root);
         let repository: Arc<dyn SnapshotRepository> = Arc::new(PosixFsSnapshotRepository::new(
             catalog_store,
             artifact_store,
+            build_files,
         ));
         let runtime_resolver: Arc<dyn SnapshotRuntimeResolver> = Arc::new(
             PosixFsRuntimeResolver::new(root, runtime_cache_root, store, cache),
@@ -111,16 +115,19 @@ impl PosixFsBackend {
 pub(crate) struct PosixFsSnapshotRepository {
     catalog_store: Arc<PosixFsCatalogStore>,
     artifact_store: Arc<PosixFsArtifactStore>,
+    build_files: Arc<PosixFsTemplateBuildFileStore>,
 }
 
 impl PosixFsSnapshotRepository {
     pub(crate) fn new(
         catalog_store: Arc<PosixFsCatalogStore>,
         artifact_store: Arc<PosixFsArtifactStore>,
+        build_files: Arc<PosixFsTemplateBuildFileStore>,
     ) -> Self {
         Self {
             catalog_store,
             artifact_store,
+            build_files,
         }
     }
 
@@ -238,6 +245,10 @@ impl SnapshotRepository for PosixFsSnapshotRepository {
         .await
     }
 
+    fn template_build_files(&self) -> Option<Arc<dyn TemplateBuildFileStore>> {
+        Some(Arc::clone(&self.build_files) as Arc<dyn TemplateBuildFileStore>)
+    }
+
     async fn publish(
         &self,
         metadata: SnapshotPublishMetadata,
@@ -400,6 +411,7 @@ mod tests {
         PosixFsSnapshotRepository::new(
             Arc::new(PosixFsCatalogStore::new(root.to_path_buf())),
             Arc::new(PosixFsArtifactStore::new(root.to_path_buf())),
+            super::super::build_files::PosixFsTemplateBuildFileStore::new(root),
         )
     }