LMDB based MerkleStore implementation

.gitignore

@@ -27,6 +27,7 @@ cli-test/vendor/
 .cursor
 .cargo/config.toml
 .claude/settings.local.json
+.claude/*.lock
 .superset
 
 data/

Cargo.toml

@@ -74,6 +74,7 @@ futures = "0.3.28"
 futures-util = "0.3.28"
 glob = "0.3.1"
 glob-match = "0.2.1"
+heed = "0.22.0"
 hex = "0.4.3"
 http = "1.1.0"
 humantime = "2.1.0"
@@ -161,6 +162,7 @@ utoipa-swagger-ui = { version = "9", features = ["actix-web"] }
 uuid = { version = "1.4.1", features = ["serde", "v4"] }
 walkdir = "2.5.0"
 xxhash-rust = { version = "0.8.7", features = ["xxh3"] }
+zerocopy = { version = "0.8", features = ["derive"] }
 zip = { version = "2.4.1", default-features = false, features = ["deflate"] }
 
 [profile.dev]

crates/lib/Cargo.toml

@@ -74,6 +74,7 @@ futures = { workspace = true }
 futures-util = { workspace = true }
 glob = { workspace = true }
 glob-match = { workspace = true }
+heed = { workspace = true }
 http = { workspace = true }
 humantime = { workspace = true }
 ignore = { workspace = true }
@@ -137,6 +138,7 @@ utoipa = { workspace = true }
 uuid = { workspace = true }
 walkdir = { workspace = true }
 xxhash-rust = { workspace = true }
+zerocopy = { workspace = true }
 zip = { workspace = true }
 
 [dev-dependencies]

crates/lib/src/core/db/merkle_node.rs

@@ -1,4 +1,8 @@
 pub mod file_backend;
+pub mod lmdb;
 pub mod merkle_node_db;
 
 pub(crate) use merkle_node_db::MerkleNodeDB;
+
+pub use file_backend::FileBackend;
+pub use lmdb::LmdbBackend;

crates/lib/src/core/db/merkle_node/file_backend.rs

@@ -354,12 +354,9 @@ fn extract_tar_under<R: Read>(
         // we have the hex-encoded hash as the `{prefix}/{suffix}` dirs.
         if let Some(hash) = extract_hash_from_entry_path(&dst_path, oxen_hidden)? {
             hashes.insert(hash);
-        } else {
-            log::warn!(
-                "Skipping non-merkle entry in tarball: {}",
-                dst_path.display()
-            );
         }
+        // If we can't extract a path, it's because we're looking at a node or children file.
+        // We will have already obtained the hash from the directory, so this is ok!
     }
     Ok(hashes)
 }