From fd9e451ff70d703408b91bb46d4b2dba378378f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Fri, 1 Dec 2023 15:58:56 +0100 Subject: [PATCH 1/9] feat: Add in-memory blockstore --- Cargo.lock | 49 +++++-- Cargo.toml | 2 +- blockstore/Cargo.toml | 14 ++ blockstore/src/in_memory_blockstore.rs | 181 +++++++++++++++++++++++++ blockstore/src/lib.rs | 21 +++ 5 files changed, 256 insertions(+), 11 deletions(-) create mode 100644 blockstore/Cargo.toml create mode 100644 blockstore/src/in_memory_blockstore.rs create mode 100644 blockstore/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 4aff50761..4a82610b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -581,6 +581,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blockstore" +version = "0.1.0" +dependencies = [ + "async-trait", + "cid 0.11.0", + "dashmap", + "multihash 0.19.1", + "tokio", +] + [[package]] name = "bs58" version = "0.5.0" @@ -752,7 +763,7 @@ dependencies = [ "bech32", "bytes", "celestia-proto", - "cid", + "cid 0.10.1", "const_format", "ed25519-consensus", "enum_dispatch", @@ -814,7 +825,19 @@ dependencies = [ "multibase", "multihash 0.18.1", "serde", - "unsigned-varint", + "unsigned-varint 0.7.2", +] + +[[package]] +name = "cid" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "472ac98592f38dfd48f188d5713a328422ed22fa39eb52b8bca495370134762a" +dependencies = [ + "core2", + "multibase", + "multihash 0.19.1", + "unsigned-varint 0.8.0", ] [[package]] @@ -2271,7 +2294,7 @@ dependencies = [ "smallvec", "thiserror", "tracing", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", ] @@ -2319,7 +2342,7 @@ dependencies = [ "sha2 0.10.8", "smallvec", "tracing", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", ] @@ -2389,7 +2412,7 @@ dependencies = [ "thiserror", "tracing", "uint", - "unsigned-varint", + "unsigned-varint 0.7.2", "void", ] @@ -2789,7 +2812,7 @@ dependencies = [ "percent-encoding", "serde", "static_assertions", - "unsigned-varint", + "unsigned-varint 0.7.2", "url", ] @@ -2812,7 +2835,7 @@ checksum = "cfd8a792c1694c6da4f68db0a9d707c72bd260994da179e6030a5dcee00bb815" dependencies = [ "core2", "multihash-derive", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -2822,7 +2845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "076d548d76a0e2a0d4ab471d0b1c36c577786dfc4471242035d97a12a735c492" dependencies = [ "core2", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -2856,7 +2879,7 @@ dependencies = [ "log", "pin-project", "smallvec", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -3471,7 +3494,7 @@ dependencies = [ "bytes", "quick-protobuf", "thiserror", - "unsigned-varint", + "unsigned-varint 0.7.2", ] [[package]] @@ -4852,6 +4875,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "unsigned-varint" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" + [[package]] name = "untrusted" version = "0.7.1" diff --git a/Cargo.toml b/Cargo.toml index 48246220c..9f951da6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["celestia", "node", "proto", "rpc", "types", "node-wasm"] +members = ["blockstore", "celestia", "node", "node-wasm", "proto", "rpc", "types"] [workspace.dependencies] celestia-node = { version = "0.1.0", path = "node" } diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml new file mode 100644 index 000000000..f299dc3f5 --- /dev/null +++ b/blockstore/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "blockstore" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +async-trait = "0.1.73" +cid = "0.11.0" +dashmap = "5.5.3" +multihash = "0.19.1" + +[dev-dependencies] +tokio = "1.29.0" diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs new file mode 100644 index 000000000..7938f3d34 --- /dev/null +++ b/blockstore/src/in_memory_blockstore.rs @@ -0,0 +1,181 @@ +use async_trait::async_trait; +use cid::CidGeneric; +use dashmap::mapref::entry::Entry; +use dashmap::DashMap; +use multihash::Multihash; + +use crate::{Blockstore, BlockstoreError, Result}; + +pub struct InMemoryBlockstore { + map: DashMap, Vec>, +} + +impl InMemoryBlockstore { + pub fn new() -> Self { + InMemoryBlockstore { + map: DashMap::new(), + } + } + + fn get_cid(&self, cid: &CidGeneric) -> Result> { + self.map + .get(cid) + .as_deref() + .cloned() + .ok_or(BlockstoreError::CidNotFound) + } + + fn insert_cid(&self, cid: CidGeneric, data: &[u8]) -> Result<()> { + let cid_entry = self.map.entry(cid); + if matches!(cid_entry, Entry::Occupied(_)) { + return Err(BlockstoreError::CidExists); + } + + cid_entry.insert(data.to_vec()); + + Ok(()) + } +} + +#[async_trait] +impl Blockstore for InMemoryBlockstore { + async fn get(&self, cid: &CidGeneric) -> Result> { + let hash = cid.hash(); + let hash = Multihash::::wrap(hash.code(), hash.digest()) + .map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::::new_v1(cid.codec(), hash); + + self.get_cid(&cid) + } + + async fn insert(&self, cid: &CidGeneric, data: &[u8]) -> Result<()> { + let hash = cid.hash(); + let hash = Multihash::::wrap(hash.code(), hash.digest()) + .map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::::new_v1(cid.codec(), hash); + + self.insert_cid(cid, data) + } +} + +impl Default for InMemoryBlockstore { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn test_insert_get() { + let store = InMemoryBlockstore::<128>::new(); + + let cid = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x01, // CID codec = 1 + 0x02, // code = 2 + 0x03, // len = 3 + 1, 2, 3, // hash + ] + .as_ref(), + ) + .unwrap(); + let data = [0xCD; 512]; + + store.insert(&cid, &data).await.unwrap(); + + let retrieved_data = store.get(&cid).await.unwrap(); + assert_eq!(data, retrieved_data.as_ref()); + + let another_cid = CidGeneric::<128>::default(); + let missing_block = store.get(&another_cid).await.unwrap_err(); + assert_eq!(missing_block, BlockstoreError::CidNotFound); + } + + #[tokio::test] + async fn test_duplicate_insert() { + let cid0 = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x02, // len = 2 + 0, 0, // hash + ] + .as_ref(), + ) + .unwrap(); + let cid1 = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x33, // CID codec + 0x44, // multihash code + 0x02, // len = 2 + 0, 1, // hash + ] + .as_ref(), + ) + .unwrap(); + + let store = InMemoryBlockstore::<128>::new(); + + store.insert(&cid0, &[0x01; 1]).await.unwrap(); + store.insert(&cid1, &[0x02; 1]).await.unwrap(); + let insert_err = store.insert(&cid1, &[0x03; 1]).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidExists); + } + + #[tokio::test] + async fn different_cid_size() { + let cid_bytes = [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x02, // len = 2 + 0, 0, // hash + ]; + let cid0 = CidGeneric::<6>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid1 = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid2 = CidGeneric::<64>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid3 = CidGeneric::<65>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid4 = CidGeneric::<128>::read_bytes(cid_bytes.as_ref()).unwrap(); + + let data = [0x99; 5]; + + let store = InMemoryBlockstore::<64>::new(); + store.insert(&cid0, data.as_ref()).await.unwrap(); + + let data0 = store.get(&cid0).await.unwrap(); + assert_eq!(data, data0.as_ref()); + let data1 = store.get(&cid1).await.unwrap(); + assert_eq!(data, data1.as_ref()); + let data2 = store.get(&cid2).await.unwrap(); + assert_eq!(data, data2.as_ref()); + let data3 = store.get(&cid3).await.unwrap(); + assert_eq!(data, data3.as_ref()); + let data4 = store.get(&cid4).await.unwrap(); + assert_eq!(data, data4.as_ref()); + } + + #[tokio::test] + async fn too_large_cid() { + let cid_bytes = [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x10, // len = 16 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ]; + let cid = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap(); + + let store = InMemoryBlockstore::<8>::new(); + let insert_err = store.insert(&cid, [0x00, 1].as_ref()).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidTooLong); + + let insert_err = store.get(&cid).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidTooLong); + } +} diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs new file mode 100644 index 000000000..32a3330f0 --- /dev/null +++ b/blockstore/src/lib.rs @@ -0,0 +1,21 @@ +use async_trait::async_trait; +use cid::CidGeneric; + +pub use in_memory_blockstore::InMemoryBlockstore; + +mod in_memory_blockstore; + +#[derive(Debug, PartialEq)] +pub enum BlockstoreError { + CidNotFound, + CidExists, + CidTooLong, +} + +type Result = std::result::Result; + +#[async_trait] +pub trait Blockstore { + async fn get(&self, cid: &CidGeneric) -> Result>; + async fn insert(&self, cid: &CidGeneric, data: &[u8]) -> Result<()>; +} From a79639fffa484ca7ae88f2af3322f19733825269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Mon, 11 Dec 2023 14:53:54 +0100 Subject: [PATCH 2/9] Align interface with fvm blockstore --- blockstore/src/in_memory_blockstore.rs | 40 ++++++++++------------ blockstore/src/lib.rs | 47 +++++++++++++++++++++++--- blockstore/src/multihash.rs | 19 +++++++++++ 3 files changed, 80 insertions(+), 26 deletions(-) create mode 100644 blockstore/src/multihash.rs diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index 7938f3d34..c84b03695 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -17,12 +17,8 @@ impl InMemoryBlockstore { } } - fn get_cid(&self, cid: &CidGeneric) -> Result> { - self.map - .get(cid) - .as_deref() - .cloned() - .ok_or(BlockstoreError::CidNotFound) + fn get_cid(&self, cid: &CidGeneric) -> Result>> { + Ok(self.map.get(cid).as_deref().cloned()) } fn insert_cid(&self, cid: CidGeneric, data: &[u8]) -> Result<()> { @@ -39,7 +35,7 @@ impl InMemoryBlockstore { #[async_trait] impl Blockstore for InMemoryBlockstore { - async fn get(&self, cid: &CidGeneric) -> Result> { + async fn get(&self, cid: &CidGeneric) -> Result>> { let hash = cid.hash(); let hash = Multihash::::wrap(hash.code(), hash.digest()) .map_err(|_| BlockstoreError::CidTooLong)?; @@ -48,7 +44,7 @@ impl Blockstore for InMemoryBlockstore { self.get_cid(&cid) } - async fn insert(&self, cid: &CidGeneric, data: &[u8]) -> Result<()> { + async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()> { let hash = cid.hash(); let hash = Multihash::::wrap(hash.code(), hash.digest()) .map_err(|_| BlockstoreError::CidTooLong)?; @@ -85,14 +81,14 @@ mod tests { .unwrap(); let data = [0xCD; 512]; - store.insert(&cid, &data).await.unwrap(); + store.put_keyed(&cid, &data).await.unwrap(); - let retrieved_data = store.get(&cid).await.unwrap(); + let retrieved_data = store.get(&cid).await.unwrap().unwrap(); assert_eq!(data, retrieved_data.as_ref()); let another_cid = CidGeneric::<128>::default(); - let missing_block = store.get(&another_cid).await.unwrap_err(); - assert_eq!(missing_block, BlockstoreError::CidNotFound); + let missing_block = store.get(&another_cid).await.unwrap(); + assert_eq!(missing_block, None); } #[tokio::test] @@ -122,9 +118,9 @@ mod tests { let store = InMemoryBlockstore::<128>::new(); - store.insert(&cid0, &[0x01; 1]).await.unwrap(); - store.insert(&cid1, &[0x02; 1]).await.unwrap(); - let insert_err = store.insert(&cid1, &[0x03; 1]).await.unwrap_err(); + store.put_keyed(&cid0, &[0x01; 1]).await.unwrap(); + store.put_keyed(&cid1, &[0x02; 1]).await.unwrap(); + let insert_err = store.put_keyed(&cid1, &[0x03; 1]).await.unwrap_err(); assert_eq!(insert_err, BlockstoreError::CidExists); } @@ -146,17 +142,17 @@ mod tests { let data = [0x99; 5]; let store = InMemoryBlockstore::<64>::new(); - store.insert(&cid0, data.as_ref()).await.unwrap(); + store.put_keyed(&cid0, data.as_ref()).await.unwrap(); - let data0 = store.get(&cid0).await.unwrap(); + let data0 = store.get(&cid0).await.unwrap().unwrap(); assert_eq!(data, data0.as_ref()); - let data1 = store.get(&cid1).await.unwrap(); + let data1 = store.get(&cid1).await.unwrap().unwrap(); assert_eq!(data, data1.as_ref()); - let data2 = store.get(&cid2).await.unwrap(); + let data2 = store.get(&cid2).await.unwrap().unwrap(); assert_eq!(data, data2.as_ref()); - let data3 = store.get(&cid3).await.unwrap(); + let data3 = store.get(&cid3).await.unwrap().unwrap(); assert_eq!(data, data3.as_ref()); - let data4 = store.get(&cid4).await.unwrap(); + let data4 = store.get(&cid4).await.unwrap().unwrap(); assert_eq!(data, data4.as_ref()); } @@ -172,7 +168,7 @@ mod tests { let cid = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap(); let store = InMemoryBlockstore::<8>::new(); - let insert_err = store.insert(&cid, [0x00, 1].as_ref()).await.unwrap_err(); + let insert_err = store.put_keyed(&cid, [0x00, 1].as_ref()).await.unwrap_err(); assert_eq!(insert_err, BlockstoreError::CidTooLong); let insert_err = store.get(&cid).await.unwrap_err(); diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs index 32a3330f0..247e7f008 100644 --- a/blockstore/src/lib.rs +++ b/blockstore/src/lib.rs @@ -1,13 +1,15 @@ use async_trait::async_trait; use cid::CidGeneric; -pub use in_memory_blockstore::InMemoryBlockstore; +use crate::multihash::Block; + +pub use crate::in_memory_blockstore::InMemoryBlockstore; mod in_memory_blockstore; +pub mod multihash; #[derive(Debug, PartialEq)] pub enum BlockstoreError { - CidNotFound, CidExists, CidTooLong, } @@ -16,6 +18,43 @@ type Result = std::result::Result; #[async_trait] pub trait Blockstore { - async fn get(&self, cid: &CidGeneric) -> Result>; - async fn insert(&self, cid: &CidGeneric, data: &[u8]) -> Result<()>; + async fn get(&self, cid: &CidGeneric) -> Result>>; + async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()>; + + async fn has(&self, cid: &CidGeneric) -> Result { + Ok(self.get(cid).await?.is_some()) + } + + async fn put(&self, block: B) -> Result<()> + where + B: Block, + { + let cid = block.cid_v1()?; + self.put_keyed(&cid, block.as_ref()).await + } + + async fn put_many(&self, blocks: I) -> Result<()> + where + B: Block, + I: IntoIterator + Send, + ::IntoIter: Send, + { + for b in blocks { + let cid = b.cid_v1()?; + self.put_keyed(&cid, b.as_ref()).await?; + } + Ok(()) + } + + async fn put_many_keyed(&self, blocks: I) -> Result<()> + where + D: AsRef<[u8]> + Send + Sync, + I: IntoIterator, D)> + Send, + ::IntoIter: Send, + { + for (cid, block) in blocks { + self.put_keyed(&cid, block.as_ref()).await?; + } + Ok(()) + } } diff --git a/blockstore/src/multihash.rs b/blockstore/src/multihash.rs new file mode 100644 index 000000000..c8248d084 --- /dev/null +++ b/blockstore/src/multihash.rs @@ -0,0 +1,19 @@ +use cid::CidGeneric; +use multihash::Multihash; + +use crate::Result; + +pub trait HasMultihash { + fn multihash(&self) -> Result>; +} + +pub trait HasCid: HasMultihash { + fn cid_v1(&self) -> Result> { + Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) + } + + fn codec() -> u64; +} + +pub trait Block: HasCid + AsRef<[u8]> + Sync + Send {} +impl + AsRef<[u8]> + Sync + Send> Block for T {} From 7554f1621112b8a550f1fd05fdcfaca7ab54a20d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Tue, 12 Dec 2023 16:53:28 +0100 Subject: [PATCH 3/9] Move CID traits to blockstore crate --- Cargo.lock | 2 ++ Cargo.toml | 1 + blockstore/Cargo.toml | 1 + blockstore/src/lib.rs | 11 ++++-- blockstore/src/multihash.rs | 23 +++++++++++-- types/Cargo.toml | 10 +++--- types/src/axis.rs | 42 ++++++++++++----------- types/src/error.rs | 5 +-- types/src/lib.rs | 1 - types/src/multihash.rs | 16 --------- types/src/namespaced_data.rs | 29 ++++++++-------- types/src/nmt.rs | 65 +++++++++++++----------------------- types/src/sample.rs | 30 ++++++++--------- types/src/share.rs | 44 +++++++++++++++++++++++- 14 files changed, 159 insertions(+), 121 deletions(-) delete mode 100644 types/src/multihash.rs diff --git a/Cargo.lock b/Cargo.lock index 6ecab95b4..50bc6ce88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -589,6 +589,7 @@ dependencies = [ "cid", "dashmap", "multihash", + "thiserror", "tokio", ] @@ -676,6 +677,7 @@ version = "0.1.0" dependencies = [ "base64 0.21.5", "bech32", + "blockstore", "bytes", "celestia-proto", "cid", diff --git a/Cargo.toml b/Cargo.toml index 1b176a49a..41aa71ce4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ libp2p = "0.53.1" nmt-rs = "0.1.0" tendermint = { git = "https://github.com/eigerco/celestia-tendermint-rs.git", rev = "bbe7de8", default-features = false } tendermint-proto = { git = "https://github.com/eigerco/celestia-tendermint-rs.git", rev = "bbe7de8" } +blockstore = { version = "0.1.0", path = "blockstore" } [patch.'https://github.com/eigerco/celestia-tendermint-rs.git'] # Uncomment to apply local changes diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml index f299dc3f5..4aed6a106 100644 --- a/blockstore/Cargo.toml +++ b/blockstore/Cargo.toml @@ -9,6 +9,7 @@ async-trait = "0.1.73" cid = "0.11.0" dashmap = "5.5.3" multihash = "0.19.1" +thiserror = "1.0.40" [dev-dependencies] tokio = "1.29.0" diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs index 247e7f008..305921d7e 100644 --- a/blockstore/src/lib.rs +++ b/blockstore/src/lib.rs @@ -1,17 +1,24 @@ use async_trait::async_trait; use cid::CidGeneric; +use thiserror::Error; -use crate::multihash::Block; +use crate::multihash::{Block, CidError}; pub use crate::in_memory_blockstore::InMemoryBlockstore; mod in_memory_blockstore; pub mod multihash; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Error)] pub enum BlockstoreError { + #[error("CID already exists in the store")] CidExists, + + #[error("CID length longer that max allowed by the store")] CidTooLong, + + #[error("Error generating CID: {0}")] + CidError(#[from] CidError), } type Result = std::result::Result; diff --git a/blockstore/src/multihash.rs b/blockstore/src/multihash.rs index c8248d084..71db7d584 100644 --- a/blockstore/src/multihash.rs +++ b/blockstore/src/multihash.rs @@ -1,14 +1,31 @@ use cid::CidGeneric; use multihash::Multihash; +use thiserror::Error; -use crate::Result; +#[derive(Debug, Error, PartialEq)] +pub enum CidError { + #[error("Invalid CID codec {0}")] + InvalidCidCodec(u64), + + #[error("Invalid multihash length {0}")] + InvalidMultihashLength(usize), + + #[error("Invalid multihash code {0} expected {1}")] + InvalidMultihashCode(u64, u64), + + #[error("Invalid data format {0}")] + InvalidDataFormat(String), + + #[error("Invalid CID: {0}")] + InvalidCid(String), +} pub trait HasMultihash { - fn multihash(&self) -> Result>; + fn multihash(&self) -> Result, CidError>; } pub trait HasCid: HasMultihash { - fn cid_v1(&self) -> Result> { + fn cid_v1(&self) -> Result, CidError> { Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) } diff --git a/types/Cargo.toml b/types/Cargo.toml index bff76db80..462643e31 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -5,10 +5,15 @@ edition = "2021" license = "Apache-2.0" [dependencies] +blockstore = { workspace = true } +celestia-proto = { workspace = true } +nmt-rs = { workspace = true } +tendermint = { workspace = true, features = ["std", "rust-crypto"] } +tendermint-proto = { workspace = true } + base64 = "0.21.2" bech32 = "0.9.1" bytes = "1.4.0" -celestia-proto = { workspace = true } cid = { version = "0.11", default-features = false, features = ["std"] } const_format = "0.2.31" ed25519-consensus = { version = "2.1.0", optional = true } @@ -16,14 +21,11 @@ enum_dispatch = "0.3.12" libp2p-identity = { version = "0.2.7", optional = true } multiaddr = { version = "0.18.0", optional = true } multihash = "0.19.1" -nmt-rs = { workspace = true } rand = { version = "0.8.5", optional = true } ruint = { version = "1.8.0", features = ["serde"] } serde = { version = "1.0.164", features = ["derive"] } serde_repr = { version = "0.1", optional = true } sha2 = "0.10.7" -tendermint = { workspace = true, features = ["std", "rust-crypto"] } -tendermint-proto = { workspace = true } thiserror = "1.0.40" [dev-dependencies] diff --git a/types/src/axis.rs b/types/src/axis.rs index e7a810e6a..0bad68c5e 100644 --- a/types/src/axis.rs +++ b/types/src/axis.rs @@ -1,12 +1,12 @@ use std::io::Cursor; use std::result::Result as StdResult; +use blockstore::multihash::{CidError, HasCid, HasMultihash}; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use sha2::{Digest, Sha256}; -use crate::multihash::{HasCid, HasMultihash}; use crate::nmt::{NamespacedHashExt, HASH_SIZE}; use crate::DataAvailabilityHeader; use crate::{Error, Result}; @@ -89,20 +89,21 @@ impl AxisId { bytes.put_u64_le(self.block_height); } - pub(crate) fn decode(buffer: &[u8]) -> Result { + pub(crate) fn decode(buffer: &[u8]) -> Result { if buffer.len() != AXIS_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let mut cursor = Cursor::new(buffer); - let axis_type = cursor.get_u8().try_into()?; + let axis_type = + AxisType::try_from(cursor.get_u8()).map_err(|e| CidError::InvalidCid(e.to_string()))?; let index = cursor.get_u16_le(); let hash = cursor.copy_to_bytes(HASH_SIZE).as_ref().try_into().unwrap(); let block_height = cursor.get_u64_le(); if block_height == 0 { - return Err(Error::ZeroBlockHeight); + return Err(CidError::InvalidCid("Zero block height".to_string())); } Ok(Self { @@ -115,7 +116,7 @@ impl AxisId { } impl HasMultihash for AxisId { - fn multihash(&self) -> Result> { + fn multihash(&self) -> Result, CidError> { let mut bytes = BytesMut::with_capacity(AXIS_ID_SIZE); self.encode(&mut bytes); // length is correct, so unwrap is safe @@ -130,24 +131,24 @@ impl HasCid for AxisId { } impl TryFrom> for AxisId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != AXIS_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, AXIS_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != AXIS_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != AXIS_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode(code, AXIS_ID_MULTIHASH_CODE)); + return Err(CidError::InvalidMultihashCode(code, AXIS_ID_MULTIHASH_CODE)); } AxisId::decode(hash.digest()) @@ -256,7 +257,10 @@ mod tests { assert_eq!(mh.code(), AXIS_ID_MULTIHASH_CODE); assert_eq!(mh.size(), AXIS_ID_SIZE as u8); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!(axis_err, Error::InvalidAxis(0xBE))); + assert_eq!( + axis_err, + CidError::InvalidCid("Invalid axis type: 190".to_string()) + ); } #[test] @@ -280,7 +284,10 @@ mod tests { assert_eq!(mh.code(), AXIS_ID_MULTIHASH_CODE); assert_eq!(mh.size(), AXIS_ID_SIZE as u8); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!(axis_err, Error::ZeroBlockHeight)); + assert_eq!( + axis_err, + CidError::InvalidCid("Zero block height".to_string()) + ); } #[test] @@ -288,10 +295,10 @@ mod tests { let multihash = Multihash::::wrap(999, &[0; AXIS_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(AXIS_ID_CODEC, multihash); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(999, AXIS_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(999, AXIS_ID_MULTIHASH_CODE) + ); } #[test] @@ -300,9 +307,6 @@ mod tests { Multihash::::wrap(AXIS_ID_MULTIHASH_CODE, &[0; AXIS_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(1234, multihash); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(1234, AXIS_ID_CODEC) - )); + assert_eq!(axis_err, CidError::InvalidCidCodec(1234)); } } diff --git a/types/src/error.rs b/types/src/error.rs index 484acc79c..e2da75dd1 100644 --- a/types/src/error.rs +++ b/types/src/error.rs @@ -64,7 +64,7 @@ pub enum Error { #[error("Invalid index of signature in commit {0}, height {1}")] InvalidSignatureIndex(usize, u64), - #[error("Invalid axis: {0}")] + #[error("Invalid axis type: {0}")] InvalidAxis(i32), #[error("Range proof verification failed: {0:?}")] @@ -111,7 +111,7 @@ pub enum Error { #[error("Invalid zero block height")] ZeroBlockHeight, - + /* #[error("Invalid multihash lenght")] InvalidMultihashLength(usize), @@ -120,6 +120,7 @@ pub enum Error { #[error("Invalid CID codec {0} expected{1}")] InvalidCidCodec(u64, u64), + */ } #[derive(Debug, thiserror::Error)] diff --git a/types/src/lib.rs b/types/src/lib.rs index f6a5a38d2..cae42c00f 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -8,7 +8,6 @@ mod error; mod extended_header; pub mod fraud_proof; pub mod hash; -pub mod multihash; mod namespaced_data; pub mod nmt; #[cfg(feature = "p2p")] diff --git a/types/src/multihash.rs b/types/src/multihash.rs deleted file mode 100644 index b5a785539..000000000 --- a/types/src/multihash.rs +++ /dev/null @@ -1,16 +0,0 @@ -use cid::CidGeneric; -use multihash::Multihash; - -use crate::Result; - -pub trait HasMultihash { - fn multihash(&self) -> Result>; -} - -pub trait HasCid: HasMultihash { - fn cid_v1(&self) -> Result> { - Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) - } - - fn codec() -> u64; -} diff --git a/types/src/namespaced_data.rs b/types/src/namespaced_data.rs index 8305c5860..a1d18deca 100644 --- a/types/src/namespaced_data.rs +++ b/types/src/namespaced_data.rs @@ -1,12 +1,12 @@ use std::io::Cursor; +use blockstore::multihash::{CidError, HasCid, HasMultihash}; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use sha2::{Digest, Sha256}; use crate::axis::AxisType; -use crate::multihash::{HasCid, HasMultihash}; use crate::nmt::{Namespace, NamespacedHashExt, HASH_SIZE, NS_SIZE}; use crate::{DataAvailabilityHeader, Error, Result}; @@ -67,9 +67,9 @@ impl NamespacedDataId { bytes.put(self.namespace.as_bytes()); } - fn decode(buffer: &[u8]) -> Result { + fn decode(buffer: &[u8]) -> Result { if buffer.len() != NAMESPACED_DATA_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let mut cursor = Cursor::new(buffer); @@ -79,7 +79,7 @@ impl NamespacedDataId { let block_height = cursor.get_u64_le(); if block_height == 0 { - return Err(Error::ZeroBlockHeight); + return Err(CidError::InvalidCid("Zero block height".to_string())); } let namespace = Namespace::from_raw(cursor.copy_to_bytes(NS_SIZE).as_ref()).unwrap(); @@ -94,7 +94,7 @@ impl NamespacedDataId { } impl HasMultihash for NamespacedDataId { - fn multihash(&self) -> Result> { + fn multihash(&self) -> Result, CidError> { let mut bytes = BytesMut::with_capacity(NAMESPACED_DATA_ID_SIZE); self.encode(&mut bytes); // length is correct, so unwrap is safe @@ -109,24 +109,24 @@ impl HasCid for NamespacedDataId { } impl TryFrom> for NamespacedDataId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != NAMESPACED_DATA_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, NAMESPACED_DATA_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != NAMESPACED_DATA_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != NAMESPACED_DATA_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode( + return Err(CidError::InvalidMultihashCode( code, NAMESPACED_DATA_ID_MULTIHASH_CODE, )); @@ -193,10 +193,10 @@ mod tests { let cid = CidGeneric::::new_v1(NAMESPACED_DATA_ID_CODEC, multihash); let axis_err = NamespacedDataId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(888, NAMESPACED_DATA_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(888, NAMESPACED_DATA_ID_MULTIHASH_CODE) + ); } #[test] @@ -208,9 +208,6 @@ mod tests { .unwrap(); let cid = CidGeneric::::new_v1(4321, multihash); let axis_err = NamespacedDataId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(4321, NAMESPACED_DATA_ID_CODEC) - )); + assert_eq!(axis_err, CidError::InvalidCidCodec(4321)); } } diff --git a/types/src/nmt.rs b/types/src/nmt.rs index 4f2eeb708..b76fa69ce 100644 --- a/types/src/nmt.rs +++ b/types/src/nmt.rs @@ -1,4 +1,5 @@ use base64::prelude::*; +use blockstore::multihash::{CidError, HasCid, HasMultihash}; use multihash::Multihash; use nmt_rs::simple_merkle::db::MemDb; use nmt_rs::simple_merkle::tree::MerkleHash; @@ -14,7 +15,6 @@ pub use self::namespace_proof::NamespaceProof; pub use self::namespaced_hash::{ NamespacedHashExt, RawNamespacedHash, HASH_SIZE, NAMESPACED_HASH_SIZE, }; -use crate::multihash::{HasCid, HasMultihash}; use crate::{Error, Result}; pub const NS_VER_SIZE: usize = 1; @@ -30,6 +30,8 @@ pub type NamespacedSha2Hasher = nmt_rs::NamespacedSha2Hasher; pub type NamespacedHash = nmt_rs::NamespacedHash; pub type Nmt = nmt_rs::NamespaceMerkleTree, NamespacedSha2Hasher, NS_SIZE>; +pub struct NodePair(NamespacedHash, NamespacedHash); + #[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] pub struct Namespace(nmt_rs::NamespaceId); @@ -176,27 +178,10 @@ impl<'de> Deserialize<'de> for Namespace { } } -impl HasMultihash for (&Namespace, &[u8]) { - fn multihash(&self) -> Result> { - let (ns, data) = self; - let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); - let namespaced_data = [ns.as_bytes(), data].concat(); - - let digest = hasher.hash_leaf(&namespaced_data).to_array(); - // size is correct, so unwrap is safe - Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) - } -} - -impl HasCid for (&Namespace, &[u8]) { - fn codec() -> u64 { - NMT_MULTIHASH_CODE - } -} +impl NodePair { + fn validate_namespace_order(&self) -> Result<()> { + let NodePair(left, right) = self; -impl HasMultihash for (&NamespacedHash, &NamespacedHash) { - fn multihash(&self) -> Result> { - let (left, right) = self; left.validate_namespace_order()?; right.validate_namespace_order()?; @@ -204,15 +189,24 @@ impl HasMultihash for (&NamespacedHash, &NamespacedHash) { return Err(Error::InvalidNmtNodeOrder); } + Ok(()) + } +} + +impl HasMultihash for NodePair { + fn multihash(&self) -> Result, CidError> { + self.validate_namespace_order() + .map_err(|e| CidError::InvalidDataFormat(e.to_string()))?; + let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); - let digest = hasher.hash_nodes(left, right).to_array(); + let digest = hasher.hash_nodes(&self.0, &self.1).to_array(); Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) } } -impl HasCid for (&NamespacedHash, &NamespacedHash) { +impl HasCid for NodePair { fn codec() -> u64 { NMT_MULTIHASH_CODE } @@ -391,20 +385,6 @@ mod tests { assert!(matches!(e, Error::UnsupportedNamespaceVersion(254))); } - #[test] - fn test_generate_leaf_multihash() { - let namespace = Namespace::new_v0(&[1, 2, 3]).unwrap(); - let data = [0xCDu8; 512]; - - let hash = (&namespace, data.as_ref()).multihash().unwrap(); - - assert_eq!(hash.code(), NMT_CODEC); - assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); - let hash = NamespacedHash::from_raw(hash.digest()).unwrap(); - assert_eq!(hash.min_namespace(), *namespace); - assert_eq!(hash.max_namespace(), *namespace); - } - #[test] fn test_generate_inner_multihash() { let ns0 = Namespace::new_v0(&[1]).unwrap(); @@ -414,7 +394,7 @@ mod tests { let left = NamespacedHash::with_min_and_max_ns(*ns0, *ns1); let right = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); - let hash = (&left, &right).multihash().unwrap(); + let hash = NodePair(left, right).multihash().unwrap(); assert_eq!(hash.code(), NMT_CODEC); assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); @@ -432,10 +412,11 @@ mod tests { let left = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); let right = NamespacedHash::with_min_and_max_ns(*ns0, *ns0); - //let multihash = Code::Sha256NamespaceFlagged; - //let digest_result = multihash.digest_nodes(&left, &right).unwrap_err(); - let result = (&left, &right).multihash().unwrap_err(); - assert!(matches!(result, Error::InvalidNmtNodeOrder)); + let result = NodePair(left, right).multihash().unwrap_err(); + assert_eq!( + result, + CidError::InvalidDataFormat("Invalid nmt node order".to_string()) + ); } #[test] diff --git a/types/src/sample.rs b/types/src/sample.rs index 812e251fe..63685cd10 100644 --- a/types/src/sample.rs +++ b/types/src/sample.rs @@ -1,11 +1,11 @@ use std::mem::size_of; +use blockstore::multihash::{CidError, HasCid, HasMultihash}; use bytes::{BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use crate::axis::{AxisId, AxisType}; -use crate::multihash::{HasCid, HasMultihash}; use crate::DataAvailabilityHeader; use crate::{Error, Result}; @@ -57,9 +57,9 @@ impl SampleId { bytes.put_u16_le(self.index); } - fn decode(buffer: &[u8]) -> Result { + fn decode(buffer: &[u8]) -> Result { if buffer.len() != SAMPLE_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let (axis_id, index) = buffer.split_at(AxisId::size()); @@ -72,7 +72,7 @@ impl SampleId { } impl HasMultihash for SampleId { - fn multihash(&self) -> Result> { + fn multihash(&self) -> Result, CidError> { let mut bytes = BytesMut::with_capacity(Self::size()); self.encode(&mut bytes); @@ -88,24 +88,27 @@ impl HasCid for SampleId { } impl TryFrom> for SampleId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != SAMPLE_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, SAMPLE_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != SAMPLE_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != SAMPLE_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode(code, SAMPLE_ID_MULTIHASH_CODE)); + return Err(CidError::InvalidMultihashCode( + code, + SAMPLE_ID_MULTIHASH_CODE, + )); } SampleId::decode(hash.digest()) @@ -183,10 +186,10 @@ mod tests { let multihash = Multihash::::wrap(888, &[0; SAMPLE_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(SAMPLE_ID_CODEC, multihash); let axis_err = SampleId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(888, SAMPLE_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(888, SAMPLE_ID_MULTIHASH_CODE) + ); } #[test] @@ -196,9 +199,6 @@ mod tests { .unwrap(); let cid = CidGeneric::::new_v1(4321, multihash); let axis_err = SampleId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(4321, SAMPLE_ID_CODEC) - )); + assert!(matches!(axis_err, CidError::InvalidCidCodec(4321))); } } diff --git a/types/src/share.rs b/types/src/share.rs index 261b4d51d..87049b848 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -1,9 +1,16 @@ +use blockstore::multihash::{CidError, HasCid, HasMultihash}; use celestia_proto::share::p2p::shrex::nd::NamespaceRowResponse as RawNamespacedRow; +use multihash::Multihash; +use nmt_rs::simple_merkle::tree::MerkleHash; +use nmt_rs::NamespaceMerkleHasher; use serde::{Deserialize, Serialize}; use tendermint_proto::Protobuf; use crate::consts::appconsts; -use crate::nmt::{Namespace, NamespaceProof, NS_SIZE}; +use crate::nmt::{ + Namespace, NamespaceProof, NamespacedSha2Hasher, NMT_CODEC, NMT_ID_SIZE, NMT_MULTIHASH_CODE, + NS_SIZE, +}; use crate::{Error, Result}; mod info_byte; @@ -70,6 +77,24 @@ impl AsRef<[u8]> for Share { } } +impl HasMultihash for Share { + fn multihash(&self) -> Result, CidError> { + //let (ns, data) = self; + let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); + //let namespaced_data = [ns.as_bytes(), data].concat(); + + let digest = hasher.hash_leaf(self.as_ref()).iter().collect::>(); + // size is correct, so unwrap is safe + Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) + } +} + +impl HasCid for Share { + fn codec() -> u64 { + NMT_MULTIHASH_CODE + } +} + #[derive(Serialize, Deserialize)] #[serde(transparent)] struct RawNamespacedShares { @@ -157,6 +182,7 @@ impl From for RawNamespacedRow { #[cfg(test)] mod tests { use super::*; + use crate::nmt::{NamespacedHash, NAMESPACED_HASH_SIZE}; use base64::prelude::*; #[cfg(target_arch = "wasm32")] @@ -330,4 +356,20 @@ mod tests { assert_eq!(ns_shares.rows[0].shares.len(), 1); assert!(!ns_shares.rows[0].proof.is_of_absence()); } + + #[test] + fn test_generate_leaf_multihash() { + let namespace = Namespace::new_v0(&[1, 2, 3]).unwrap(); + let mut data = [0xCDu8; appconsts::SHARE_SIZE]; + data[..NS_SIZE].copy_from_slice(namespace.as_bytes()); + let share = Share::from_raw(&data).unwrap(); + + let hash = share.multihash().unwrap(); + + assert_eq!(hash.code(), NMT_CODEC); + assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); + let hash = NamespacedHash::try_from(hash.digest()).unwrap(); + assert_eq!(hash.min_namespace(), *namespace); + assert_eq!(hash.max_namespace(), *namespace); + } } From b0ba48ce778356fea2da8ba9ab2ab533a9817fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Tue, 12 Dec 2023 21:54:30 +0100 Subject: [PATCH 4/9] Add docs --- blockstore/Cargo.toml | 3 +++ blockstore/src/in_memory_blockstore.rs | 5 ++-- blockstore/src/lib.rs | 34 ++++++++++++++++++++++++-- blockstore/src/multihash.rs | 17 +++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml index 4aed6a106..34f82d04e 100644 --- a/blockstore/Cargo.toml +++ b/blockstore/Cargo.toml @@ -13,3 +13,6 @@ thiserror = "1.0.40" [dev-dependencies] tokio = "1.29.0" + +[package.metadata.docs.rs] +rustdoc-args = ["--cfg", "docs_rs"] diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index c84b03695..8078494c0 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -1,4 +1,3 @@ -use async_trait::async_trait; use cid::CidGeneric; use dashmap::mapref::entry::Entry; use dashmap::DashMap; @@ -6,11 +5,13 @@ use multihash::Multihash; use crate::{Blockstore, BlockstoreError, Result}; +/// Simple in-memory blockstore implementation. pub struct InMemoryBlockstore { map: DashMap, Vec>, } impl InMemoryBlockstore { + /// Create new empty in-memory blockstore pub fn new() -> Self { InMemoryBlockstore { map: DashMap::new(), @@ -33,7 +34,7 @@ impl InMemoryBlockstore { } } -#[async_trait] +#[cfg_attr(not(docs_rs), async_trait::async_trait)] impl Blockstore for InMemoryBlockstore { async fn get(&self, cid: &CidGeneric) -> Result>> { let hash = cid.hash(); diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs index 305921d7e..012bb763d 100644 --- a/blockstore/src/lib.rs +++ b/blockstore/src/lib.rs @@ -1,4 +1,5 @@ -use async_trait::async_trait; +#![cfg_attr(docs_rs, feature(async_fn_in_trait))] + use cid::CidGeneric; use thiserror::Error; @@ -7,31 +8,55 @@ use crate::multihash::{Block, CidError}; pub use crate::in_memory_blockstore::InMemoryBlockstore; mod in_memory_blockstore; +/// Utilities related to computing CID for the inserted data pub mod multihash; +/// Error returned when performing operations on [`Blockstore`] #[derive(Debug, PartialEq, Error)] pub enum BlockstoreError { + /// Provided CID already exists in blockstore when trying to insert it #[error("CID already exists in the store")] CidExists, + /// Provided CID is longer than max length supported by the blockstore #[error("CID length longer that max allowed by the store")] CidTooLong, + /// Error occured when trying to compute CID with [`HasCid`] trait + /// + /// [`HasCid`]: crate::multihash::HasCid #[error("Error generating CID: {0}")] CidError(#[from] CidError), } type Result = std::result::Result; -#[async_trait] +/// A IPLD blockstore capable of holding arbitrary data indexed by CID. Implementations can impose +/// limit on supported CID length, and any operations on longer CIDs will fail with [`CidTooLong`] +/// +/// [`CidTooLong`]: BlockstoreError::CidTooLong + +#[cfg_attr(not(docs_rs), async_trait::async_trait)] pub trait Blockstore { + /// Gets the block from the blockstore async fn get(&self, cid: &CidGeneric) -> Result>>; + + /// Inserts the data with pre-computed CID. + /// Use [`put`], if you want CID to be computed. + /// + /// [`put`]: Blockstore::put async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()>; + /// Checks whether blockstore has block for provided CID async fn has(&self, cid: &CidGeneric) -> Result { Ok(self.get(cid).await?.is_some()) } + /// Inserts the data into the blockstore, computing CID using [`HasCid`] trait that needs to be + /// implemented for [`Block`] + /// + /// [`Block`]: crate::multihash::Block + /// [`HasCid`]: crate::multihash::HasCid async fn put(&self, block: B) -> Result<()> where B: Block, @@ -40,6 +65,9 @@ pub trait Blockstore { self.put_keyed(&cid, block.as_ref()).await } + /// Inserts multiple blocks into the blockstore computing their CID + /// If CID computation, or insert itself fails, error is returned and subsequent items are also + /// skipped. async fn put_many(&self, blocks: I) -> Result<()> where B: Block, @@ -53,6 +81,8 @@ pub trait Blockstore { Ok(()) } + /// Inserts multiple blocks with pre-computed CID into the blockstore. + /// If any put from the list fails, error is returned and subsequent items are also skipped. async fn put_many_keyed(&self, blocks: I) -> Result<()> where D: AsRef<[u8]> + Send + Sync, diff --git a/blockstore/src/multihash.rs b/blockstore/src/multihash.rs index 71db7d584..7b338fc88 100644 --- a/blockstore/src/multihash.rs +++ b/blockstore/src/multihash.rs @@ -2,28 +2,37 @@ use cid::CidGeneric; use multihash::Multihash; use thiserror::Error; +/// Error returned when trying to compute new or parse existing CID. Note that errors here can be +/// specific to particular [`Block`] impl, and don't necessarily indicate invalid CID in general. #[derive(Debug, Error, PartialEq)] pub enum CidError { + /// Coded specified in CID is not supported in this context #[error("Invalid CID codec {0}")] InvalidCidCodec(u64), + /// CID's multihash length is different from expected #[error("Invalid multihash length {0}")] InvalidMultihashLength(usize), + /// Encountered multihash code is not supported in this context #[error("Invalid multihash code {0} expected {1}")] InvalidMultihashCode(u64, u64), + /// CID could not be computed for the provided data due to some constraint violation #[error("Invalid data format {0}")] InvalidDataFormat(String), + /// CID is well-formed but contains invalid data #[error("Invalid CID: {0}")] InvalidCid(String), } +/// Trait indicating a struct that can compute its own multihash pub trait HasMultihash { fn multihash(&self) -> Result, CidError>; } +/// Trait indicating a struct that can compute its own CID pub trait HasCid: HasMultihash { fn cid_v1(&self) -> Result, CidError> { Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) @@ -32,5 +41,13 @@ pub trait HasCid: HasMultihash { fn codec() -> u64; } +/// Trait for structs that can be inserted into the [`Blockstore`]. For this purpose they need to +/// know how to compute their own CID ([`HasCid`] trait) as well as have a byte string +/// representation (`AsRef[u8]` trait). If CID is known, or it's otherwise more conventient, one +/// can use [`Blockstore::put_keyed`] with `CID` and `&[u8]` directly. +/// +/// [`Blockstore`]: crate::Blockstore +/// [`Blockstore::put_keyed`]: crate::Blockstore::put_keyed pub trait Block: HasCid + AsRef<[u8]> + Sync + Send {} + impl + AsRef<[u8]> + Sync + Send> Block for T {} From f94d3240a933dd8f5f10bb1d77f9006668538a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Wed, 13 Dec 2023 09:55:51 +0100 Subject: [PATCH 5/9] Add tests, cleanup --- blockstore/Cargo.toml | 2 +- blockstore/src/in_memory_blockstore.rs | 113 +++++++++++++++++++++++-- types/src/error.rs | 10 --- types/src/share.rs | 4 +- 4 files changed, 107 insertions(+), 22 deletions(-) diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml index 34f82d04e..b684e2f19 100644 --- a/blockstore/Cargo.toml +++ b/blockstore/Cargo.toml @@ -12,7 +12,7 @@ multihash = "0.19.1" thiserror = "1.0.40" [dev-dependencies] -tokio = "1.29.0" +tokio = { version = "1.29.0", features = [ "macros", "rt" ] } [package.metadata.docs.rs] rustdoc-args = ["--cfg", "docs_rs"] diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index 8078494c0..7b1b4bf31 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -64,6 +64,9 @@ impl Default for InMemoryBlockstore { #[cfg(test)] mod tests { use super::*; + use crate::multihash::{CidError, HasCid, HasMultihash}; + use std::iter::zip; + use std::result::Result as StdResult; #[tokio::test] async fn test_insert_get() { @@ -159,14 +162,17 @@ mod tests { #[tokio::test] async fn too_large_cid() { - let cid_bytes = [ - 0x01, // CIDv1 - 0x11, // CID codec - 0x22, // multihash code - 0x10, // len = 16 - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]; - let cid = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid = CidGeneric::<32>::read_bytes( + [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x10, // len = 16 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] + .as_ref(), + ) + .unwrap(); let store = InMemoryBlockstore::<8>::new(); let insert_err = store.put_keyed(&cid, [0x00, 1].as_ref()).await.unwrap_err(); @@ -175,4 +181,95 @@ mod tests { let insert_err = store.get(&cid).await.unwrap_err(); assert_eq!(insert_err, BlockstoreError::CidTooLong); } + + #[tokio::test] + async fn test_block_insert() { + let block = TestBlock([0, 1, 2, 3]); + + let store = InMemoryBlockstore::<8>::new(); + store.put(block).await.unwrap(); + let retrieved_block = store.get(&block.cid_v1().unwrap()).await.unwrap().unwrap(); + assert_eq!(block.as_ref(), &retrieved_block); + } + + #[tokio::test] + async fn test_multiple_blocks_insert() { + let blocks = [ + TestBlock([0, 0, 0, 0]), + TestBlock([0, 0, 0, 1]), + TestBlock([0, 0, 1, 0]), + TestBlock([0, 0, 1, 1]), + TestBlock([0, 1, 0, 0]), + TestBlock([0, 1, 0, 1]), + TestBlock([0, 1, 1, 0]), + TestBlock([0, 1, 1, 1]), + ]; + let uninserted_blocks = [ + TestBlock([1, 0, 0, 0]), + TestBlock([1, 0, 0, 1]), + TestBlock([1, 0, 1, 0]), + TestBlock([1, 1, 0, 1]), + ]; + + let store = InMemoryBlockstore::<8>::new(); + store.put_many(blocks).await.unwrap(); + + for b in blocks { + let cid = b.cid_v1().unwrap(); + assert!(store.has(&cid).await.unwrap()); + let retrieved_block = store.get(&cid).await.unwrap().unwrap(); + assert_eq!(b.as_ref(), &retrieved_block); + } + + for b in uninserted_blocks { + let cid = b.cid_v1().unwrap(); + assert!(!store.has(&cid).await.unwrap()); + assert!(store.get(&cid).await.unwrap().is_none()); + } + } + + #[tokio::test] + async fn test_multiple_keyed() { + let blocks = [[0], [1], [2], [3]]; + let cids = [ + // 4 different arbitrary CIDs + TestBlock([0, 0, 0, 1]).cid_v1().unwrap(), + TestBlock([0, 0, 0, 2]).cid_v1().unwrap(), + TestBlock([0, 0, 0, 3]).cid_v1().unwrap(), + TestBlock([0, 0, 0, 4]).cid_v1().unwrap(), + ]; + let pairs = zip(cids, blocks); + + let store = InMemoryBlockstore::<8>::new(); + store.put_many_keyed(pairs.clone()).await.unwrap(); + + for (cid, block) in pairs { + let retrieved_block = store.get(&cid).await.unwrap().unwrap(); + assert_eq!(block.as_ref(), &retrieved_block); + } + } + + const TEST_CODEC: u64 = 0x0A; + const TEST_MH_CODE: u64 = 0x0A; + + #[derive(Debug, PartialEq, Clone, Copy)] + struct TestBlock(pub [u8; 4]); + + impl HasMultihash<8> for TestBlock { + fn multihash(&self) -> StdResult, CidError> { + Ok(Multihash::wrap(TEST_MH_CODE, &self.0).unwrap()) + } + } + + impl HasCid<8> for TestBlock { + fn codec() -> u64 { + TEST_CODEC + } + } + + impl AsRef<[u8]> for TestBlock { + fn as_ref(&self) -> &[u8] { + &self.0 + } + } } diff --git a/types/src/error.rs b/types/src/error.rs index e2da75dd1..3f1423bb3 100644 --- a/types/src/error.rs +++ b/types/src/error.rs @@ -111,16 +111,6 @@ pub enum Error { #[error("Invalid zero block height")] ZeroBlockHeight, - /* - #[error("Invalid multihash lenght")] - InvalidMultihashLength(usize), - - #[error("Invalid multihash code {0} expected {1}")] - InvalidMultihashCode(u64, u64), - - #[error("Invalid CID codec {0} expected{1}")] - InvalidCidCodec(u64, u64), - */ } #[derive(Debug, thiserror::Error)] diff --git a/types/src/share.rs b/types/src/share.rs index 87049b848..db7209a7e 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -79,11 +79,9 @@ impl AsRef<[u8]> for Share { impl HasMultihash for Share { fn multihash(&self) -> Result, CidError> { - //let (ns, data) = self; let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); - //let namespaced_data = [ns.as_bytes(), data].concat(); - let digest = hasher.hash_leaf(self.as_ref()).iter().collect::>(); + // size is correct, so unwrap is safe Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) } From be8d7762af2f8dd7dae0f8e8d400d1d531441257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Wed, 13 Dec 2023 13:11:11 +0100 Subject: [PATCH 6/9] rename const generic to sth more verbose --- blockstore/src/in_memory_blockstore.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index 7b1b4bf31..e34aeda74 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -6,11 +6,11 @@ use multihash::Multihash; use crate::{Blockstore, BlockstoreError, Result}; /// Simple in-memory blockstore implementation. -pub struct InMemoryBlockstore { - map: DashMap, Vec>, +pub struct InMemoryBlockstore { + map: DashMap, Vec>, } -impl InMemoryBlockstore { +impl InMemoryBlockstore { /// Create new empty in-memory blockstore pub fn new() -> Self { InMemoryBlockstore { @@ -18,11 +18,11 @@ impl InMemoryBlockstore { } } - fn get_cid(&self, cid: &CidGeneric) -> Result>> { + fn get_cid(&self, cid: &CidGeneric) -> Result>> { Ok(self.map.get(cid).as_deref().cloned()) } - fn insert_cid(&self, cid: CidGeneric, data: &[u8]) -> Result<()> { + fn insert_cid(&self, cid: CidGeneric, data: &[u8]) -> Result<()> { let cid_entry = self.map.entry(cid); if matches!(cid_entry, Entry::Occupied(_)) { return Err(BlockstoreError::CidExists); @@ -35,27 +35,27 @@ impl InMemoryBlockstore { } #[cfg_attr(not(docs_rs), async_trait::async_trait)] -impl Blockstore for InMemoryBlockstore { +impl Blockstore for InMemoryBlockstore { async fn get(&self, cid: &CidGeneric) -> Result>> { let hash = cid.hash(); - let hash = Multihash::::wrap(hash.code(), hash.digest()) - .map_err(|_| BlockstoreError::CidTooLong)?; - let cid = CidGeneric::::new_v1(cid.codec(), hash); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::new_v1(cid.codec(), hash); self.get_cid(&cid) } async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()> { let hash = cid.hash(); - let hash = Multihash::::wrap(hash.code(), hash.digest()) - .map_err(|_| BlockstoreError::CidTooLong)?; - let cid = CidGeneric::::new_v1(cid.codec(), hash); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::new_v1(cid.codec(), hash); self.insert_cid(cid, data) } } -impl Default for InMemoryBlockstore { +impl Default for InMemoryBlockstore { fn default() -> Self { Self::new() } From 09e16a640871f7472fc4bcd520ce2a2e7331cba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Fri, 15 Dec 2023 09:55:46 +0100 Subject: [PATCH 7/9] Update interface --- blockstore/src/in_memory_blockstore.rs | 37 +++++++++------------- blockstore/src/lib.rs | 8 ++--- blockstore/src/multihash.rs | 29 +++-------------- types/src/axis.rs | 34 ++++++++++---------- types/src/namespaced_data.rs | 34 ++++++++++---------- types/src/nmt.rs | 44 +++++++++++++++----------- types/src/sample.rs | 36 ++++++++++----------- types/src/share.rs | 24 +++++++------- 8 files changed, 111 insertions(+), 135 deletions(-) diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index e34aeda74..cc18654d3 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -64,7 +64,7 @@ impl Default for InMemoryBlockstore::new(); store.put(block).await.unwrap(); - let retrieved_block = store.get(&block.cid_v1().unwrap()).await.unwrap().unwrap(); - assert_eq!(block.as_ref(), &retrieved_block); + let retrieved_block = store.get(&block.cid().unwrap()).await.unwrap().unwrap(); + assert_eq!(block.data(), &retrieved_block); } #[tokio::test] @@ -215,14 +215,14 @@ mod tests { store.put_many(blocks).await.unwrap(); for b in blocks { - let cid = b.cid_v1().unwrap(); + let cid = b.cid().unwrap(); assert!(store.has(&cid).await.unwrap()); let retrieved_block = store.get(&cid).await.unwrap().unwrap(); - assert_eq!(b.as_ref(), &retrieved_block); + assert_eq!(b.data(), &retrieved_block); } for b in uninserted_blocks { - let cid = b.cid_v1().unwrap(); + let cid = b.cid().unwrap(); assert!(!store.has(&cid).await.unwrap()); assert!(store.get(&cid).await.unwrap().is_none()); } @@ -233,10 +233,10 @@ mod tests { let blocks = [[0], [1], [2], [3]]; let cids = [ // 4 different arbitrary CIDs - TestBlock([0, 0, 0, 1]).cid_v1().unwrap(), - TestBlock([0, 0, 0, 2]).cid_v1().unwrap(), - TestBlock([0, 0, 0, 3]).cid_v1().unwrap(), - TestBlock([0, 0, 0, 4]).cid_v1().unwrap(), + TestBlock([0, 0, 0, 1]).cid().unwrap(), + TestBlock([0, 0, 0, 2]).cid().unwrap(), + TestBlock([0, 0, 0, 3]).cid().unwrap(), + TestBlock([0, 0, 0, 4]).cid().unwrap(), ]; let pairs = zip(cids, blocks); @@ -255,20 +255,13 @@ mod tests { #[derive(Debug, PartialEq, Clone, Copy)] struct TestBlock(pub [u8; 4]); - impl HasMultihash<8> for TestBlock { - fn multihash(&self) -> StdResult, CidError> { - Ok(Multihash::wrap(TEST_MH_CODE, &self.0).unwrap()) + impl Block<8> for TestBlock { + fn cid(&self) -> StdResult, CidError> { + let mh = Multihash::wrap(TEST_MH_CODE, &self.0).unwrap(); + Ok(CidGeneric::new_v1(TEST_CODEC, mh)) } - } - - impl HasCid<8> for TestBlock { - fn codec() -> u64 { - TEST_CODEC - } - } - impl AsRef<[u8]> for TestBlock { - fn as_ref(&self) -> &[u8] { + fn data(&self) -> &[u8] { &self.0 } } diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs index 012bb763d..f14a2ba94 100644 --- a/blockstore/src/lib.rs +++ b/blockstore/src/lib.rs @@ -61,8 +61,8 @@ pub trait Blockstore { where B: Block, { - let cid = block.cid_v1()?; - self.put_keyed(&cid, block.as_ref()).await + let cid = block.cid()?; + self.put_keyed(&cid, block.data()).await } /// Inserts multiple blocks into the blockstore computing their CID @@ -75,8 +75,8 @@ pub trait Blockstore { ::IntoIter: Send, { for b in blocks { - let cid = b.cid_v1()?; - self.put_keyed(&cid, b.as_ref()).await?; + let cid = b.cid()?; + self.put_keyed(&cid, b.data()).await?; } Ok(()) } diff --git a/blockstore/src/multihash.rs b/blockstore/src/multihash.rs index 7b338fc88..447b7429f 100644 --- a/blockstore/src/multihash.rs +++ b/blockstore/src/multihash.rs @@ -1,5 +1,4 @@ use cid::CidGeneric; -use multihash::Multihash; use thiserror::Error; /// Error returned when trying to compute new or parse existing CID. Note that errors here can be @@ -27,27 +26,9 @@ pub enum CidError { InvalidCid(String), } -/// Trait indicating a struct that can compute its own multihash -pub trait HasMultihash { - fn multihash(&self) -> Result, CidError>; +/// Trait for a block of data which can be represented as a string of bytes, which can compute its +/// own CID +pub trait Block: Sync + Send { + fn cid(&self) -> Result, CidError>; + fn data(&self) -> &[u8]; } - -/// Trait indicating a struct that can compute its own CID -pub trait HasCid: HasMultihash { - fn cid_v1(&self) -> Result, CidError> { - Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) - } - - fn codec() -> u64; -} - -/// Trait for structs that can be inserted into the [`Blockstore`]. For this purpose they need to -/// know how to compute their own CID ([`HasCid`] trait) as well as have a byte string -/// representation (`AsRef[u8]` trait). If CID is known, or it's otherwise more conventient, one -/// can use [`Blockstore::put_keyed`] with `CID` and `&[u8]` directly. -/// -/// [`Blockstore`]: crate::Blockstore -/// [`Blockstore::put_keyed`]: crate::Blockstore::put_keyed -pub trait Block: HasCid + AsRef<[u8]> + Sync + Send {} - -impl + AsRef<[u8]> + Sync + Send> Block for T {} diff --git a/types/src/axis.rs b/types/src/axis.rs index 0bad68c5e..95562c194 100644 --- a/types/src/axis.rs +++ b/types/src/axis.rs @@ -1,7 +1,7 @@ use std::io::Cursor; use std::result::Result as StdResult; -use blockstore::multihash::{CidError, HasCid, HasMultihash}; +use blockstore::multihash::CidError; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; @@ -37,7 +37,7 @@ impl TryFrom for AxisType { /// Represents particular particular Column or Row in a specific Data Square, /// paired together with a hash of the axis root. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct AxisId { pub axis_type: AxisType, pub index: u16, @@ -115,21 +115,6 @@ impl AxisId { } } -impl HasMultihash for AxisId { - fn multihash(&self) -> Result, CidError> { - let mut bytes = BytesMut::with_capacity(AXIS_ID_SIZE); - self.encode(&mut bytes); - // length is correct, so unwrap is safe - Ok(Multihash::wrap(AXIS_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for AxisId { - fn codec() -> u64 { - AXIS_ID_CODEC - } -} - impl TryFrom> for AxisId { type Error = CidError; @@ -155,6 +140,19 @@ impl TryFrom> for AxisId { } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(axis: AxisId) -> Result { + let mut bytes = BytesMut::with_capacity(AXIS_ID_SIZE); + axis.encode(&mut bytes); + // length is correct, so unwrap is safe + let mh = Multihash::wrap(AXIS_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(AXIS_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -184,7 +182,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let axis_id = AxisId::new(AxisType::Row, 5, &dah, 100).unwrap(); - let cid = axis_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(axis_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), AXIS_ID_MULTIHASH_CODE); diff --git a/types/src/namespaced_data.rs b/types/src/namespaced_data.rs index a1d18deca..532220be1 100644 --- a/types/src/namespaced_data.rs +++ b/types/src/namespaced_data.rs @@ -1,6 +1,6 @@ use std::io::Cursor; -use blockstore::multihash::{CidError, HasCid, HasMultihash}; +use blockstore::multihash::CidError; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; @@ -15,7 +15,7 @@ pub const NAMESPACED_DATA_ID_MULTIHASH_CODE: u64 = 0x7821; pub const NAMESPACED_DATA_ID_CODEC: u64 = 0x7820; /// Represents shares from a namespace located on a particular row of Data Square -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct NamespacedDataId { pub namespace: Namespace, pub row_index: u16, @@ -93,21 +93,6 @@ impl NamespacedDataId { } } -impl HasMultihash for NamespacedDataId { - fn multihash(&self) -> Result, CidError> { - let mut bytes = BytesMut::with_capacity(NAMESPACED_DATA_ID_SIZE); - self.encode(&mut bytes); - // length is correct, so unwrap is safe - Ok(Multihash::wrap(NAMESPACED_DATA_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for NamespacedDataId { - fn codec() -> u64 { - NAMESPACED_DATA_ID_CODEC - } -} - impl TryFrom> for NamespacedDataId { type Error = CidError; @@ -136,6 +121,19 @@ impl TryFrom> for NamespacedDataId { } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(namespaced_data_id: NamespacedDataId) -> Result { + let mut bytes = BytesMut::with_capacity(NAMESPACED_DATA_ID_SIZE); + namespaced_data_id.encode(&mut bytes); + // length is correct, so unwrap is safe + let mh = Multihash::wrap(NAMESPACED_DATA_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(NAMESPACED_DATA_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -149,7 +147,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let data_id = NamespacedDataId::new(ns, 5, &dah, 100).unwrap(); - let cid = data_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(data_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), NAMESPACED_DATA_ID_MULTIHASH_CODE); diff --git a/types/src/nmt.rs b/types/src/nmt.rs index b76fa69ce..b43bd4022 100644 --- a/types/src/nmt.rs +++ b/types/src/nmt.rs @@ -1,5 +1,6 @@ use base64::prelude::*; -use blockstore::multihash::{CidError, HasCid, HasMultihash}; +use blockstore::multihash::CidError; +use cid::CidGeneric; use multihash::Multihash; use nmt_rs::simple_merkle::db::MemDb; use nmt_rs::simple_merkle::tree::MerkleHash; @@ -193,22 +194,20 @@ impl NodePair { } } -impl HasMultihash for NodePair { - fn multihash(&self) -> Result, CidError> { - self.validate_namespace_order() +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(nodes: NodePair) -> Result { + nodes + .validate_namespace_order() .map_err(|e| CidError::InvalidDataFormat(e.to_string()))?; let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); + let digest = hasher.hash_nodes(&nodes.0, &nodes.1).to_array(); - let digest = hasher.hash_nodes(&self.0, &self.1).to_array(); - - Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) - } -} + let mh = Multihash::wrap(NMT_MULTIHASH_CODE, &digest).unwrap(); -impl HasCid for NodePair { - fn codec() -> u64 { - NMT_MULTIHASH_CODE + Ok(CidGeneric::new_v1(NMT_CODEC, mh)) } } @@ -391,13 +390,18 @@ mod tests { let ns1 = Namespace::new_v0(&[2]).unwrap(); let ns2 = Namespace::new_v0(&[3]).unwrap(); - let left = NamespacedHash::with_min_and_max_ns(*ns0, *ns1); - let right = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); + let nodes = NodePair( + NamespacedHash::with_min_and_max_ns(*ns0, *ns1), + NamespacedHash::with_min_and_max_ns(*ns1, *ns2), + ); - let hash = NodePair(left, right).multihash().unwrap(); + let cid = CidGeneric::try_from(nodes).unwrap(); + assert_eq!(cid.codec(), NMT_CODEC); - assert_eq!(hash.code(), NMT_CODEC); + let hash = cid.hash(); + assert_eq!(hash.code(), NMT_MULTIHASH_CODE); assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); + let hash = NamespacedHash::from_raw(hash.digest()).unwrap(); assert_eq!(hash.min_namespace(), *ns0); assert_eq!(hash.max_namespace(), *ns2); @@ -409,10 +413,12 @@ mod tests { let ns1 = Namespace::new_v0(&[2]).unwrap(); let ns2 = Namespace::new_v0(&[3]).unwrap(); - let left = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); - let right = NamespacedHash::with_min_and_max_ns(*ns0, *ns0); + let nodes = NodePair( + NamespacedHash::with_min_and_max_ns(*ns1, *ns2), + NamespacedHash::with_min_and_max_ns(*ns0, *ns0), + ); + let result = CidGeneric::try_from(nodes).unwrap_err(); - let result = NodePair(left, right).multihash().unwrap_err(); assert_eq!( result, CidError::InvalidDataFormat("Invalid nmt node order".to_string()) diff --git a/types/src/sample.rs b/types/src/sample.rs index 63685cd10..75a22b705 100644 --- a/types/src/sample.rs +++ b/types/src/sample.rs @@ -1,6 +1,6 @@ use std::mem::size_of; -use blockstore::multihash::{CidError, HasCid, HasMultihash}; +use blockstore::multihash::CidError; use bytes::{BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; @@ -14,7 +14,7 @@ pub const SAMPLE_ID_MULTIHASH_CODE: u64 = 0x7801; pub const SAMPLE_ID_CODEC: u64 = 0x7800; /// Represents particular sample along the axis on specific Data Square -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct SampleId { pub axis: AxisId, pub index: u16, @@ -71,22 +71,6 @@ impl SampleId { } } -impl HasMultihash for SampleId { - fn multihash(&self) -> Result, CidError> { - let mut bytes = BytesMut::with_capacity(Self::size()); - - self.encode(&mut bytes); - - Ok(Multihash::::wrap(SAMPLE_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for SampleId { - fn codec() -> u64 { - SAMPLE_ID_CODEC - } -} - impl TryFrom> for SampleId { type Error = CidError; @@ -115,6 +99,20 @@ impl TryFrom> for SampleId { } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(sample_id: SampleId) -> Result { + let mut bytes = BytesMut::with_capacity(SAMPLE_ID_SIZE); + // length is correct, so unwrap is safe + sample_id.encode(&mut bytes); + + let mh = Multihash::wrap(SAMPLE_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(SAMPLE_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -127,7 +125,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let sample_id = SampleId::new(AxisType::Row, 5, &dah, 100).unwrap(); - let cid = sample_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(sample_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), SAMPLE_ID_MULTIHASH_CODE); diff --git a/types/src/share.rs b/types/src/share.rs index db7209a7e..9515d2456 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -1,5 +1,6 @@ -use blockstore::multihash::{CidError, HasCid, HasMultihash}; +use blockstore::multihash::{Block, CidError}; use celestia_proto::share::p2p::shrex::nd::NamespaceRowResponse as RawNamespacedRow; +use cid::CidGeneric; use multihash::Multihash; use nmt_rs::simple_merkle::tree::MerkleHash; use nmt_rs::NamespaceMerkleHasher; @@ -77,19 +78,19 @@ impl AsRef<[u8]> for Share { } } -impl HasMultihash for Share { - fn multihash(&self) -> Result, CidError> { +impl Block for Share { + fn cid(&self) -> Result, CidError> { let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); let digest = hasher.hash_leaf(self.as_ref()).iter().collect::>(); // size is correct, so unwrap is safe - Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) + let mh = Multihash::wrap(NMT_MULTIHASH_CODE, &digest).unwrap(); + + Ok(CidGeneric::new_v1(NMT_CODEC, mh)) } -} -impl HasCid for Share { - fn codec() -> u64 { - NMT_MULTIHASH_CODE + fn data(&self) -> &[u8] { + &self.data } } @@ -362,9 +363,10 @@ mod tests { data[..NS_SIZE].copy_from_slice(namespace.as_bytes()); let share = Share::from_raw(&data).unwrap(); - let hash = share.multihash().unwrap(); - - assert_eq!(hash.code(), NMT_CODEC); + let cid = share.cid().unwrap(); + assert_eq!(cid.codec(), NMT_CODEC); + let hash = cid.hash(); + assert_eq!(hash.code(), NMT_MULTIHASH_CODE); assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); let hash = NamespacedHash::try_from(hash.digest()).unwrap(); assert_eq!(hash.min_namespace(), *namespace); From 2d2db785473dfab38611b40c12eab93dc0a29a98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Florkiewicz?= Date: Fri, 15 Dec 2023 10:06:52 +0100 Subject: [PATCH 8/9] Organise --- blockstore/src/{multihash.rs => block.rs} | 0 blockstore/src/in_memory_blockstore.rs | 2 +- blockstore/src/lib.rs | 6 +++--- types/src/axis.rs | 2 +- types/src/namespaced_data.rs | 2 +- types/src/nmt.rs | 2 +- types/src/sample.rs | 2 +- types/src/share.rs | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) rename blockstore/src/{multihash.rs => block.rs} (100%) diff --git a/blockstore/src/multihash.rs b/blockstore/src/block.rs similarity index 100% rename from blockstore/src/multihash.rs rename to blockstore/src/block.rs diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index cc18654d3..6a9891e9e 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -64,7 +64,7 @@ impl Default for InMemoryBlockstore Date: Fri, 15 Dec 2023 14:08:13 +0100 Subject: [PATCH 9/9] add specialisation --- blockstore/src/in_memory_blockstore.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs index 6a9891e9e..03e6c0124 100644 --- a/blockstore/src/in_memory_blockstore.rs +++ b/blockstore/src/in_memory_blockstore.rs @@ -32,6 +32,10 @@ impl InMemoryBlockstore { Ok(()) } + + fn contains_cid(&self, cid: &CidGeneric) -> bool { + self.map.contains_key(cid) + } } #[cfg_attr(not(docs_rs), async_trait::async_trait)] @@ -53,6 +57,11 @@ impl Blockstore for InMemoryBlockstore(&self, cid: &CidGeneric) -> Result { + let cid = get_internal_cid(cid)?; + Ok(self.contains_cid(&cid)) + } } impl Default for InMemoryBlockstore { @@ -61,6 +70,15 @@ impl Default for InMemoryBlockstore( + cid: &CidGeneric, +) -> Result> { + let hash = cid.hash(); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + Ok(CidGeneric::new_v1(cid.codec(), hash)) +} + #[cfg(test)] mod tests { use super::*;