diff --git a/Cargo.lock b/Cargo.lock index 62c3eeb71..3c65c8097 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -581,6 +581,18 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blockstore" +version = "0.1.0" +dependencies = [ + "async-trait", + "cid", + "dashmap", + "multihash", + "thiserror", + "tokio", +] + [[package]] name = "bs58" version = "0.5.0" @@ -666,6 +678,7 @@ version = "0.1.0" dependencies = [ "base64 0.21.5", "bech32", + "blockstore", "bytes", "celestia-proto", "cid", diff --git a/Cargo.toml b/Cargo.toml index 19a9c2d37..41aa71ce4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["cli", "node", "node-wasm", "proto", "rpc", "types"] +members = ["blockstore", "cli", "node", "node-wasm", "proto", "rpc", "types"] [workspace.dependencies] lumina-node = { version = "0.1.0", path = "node" } @@ -12,6 +12,7 @@ libp2p = "0.53.1" nmt-rs = "0.1.0" tendermint = { git = "https://github.com/eigerco/celestia-tendermint-rs.git", rev = "bbe7de8", default-features = false } tendermint-proto = { git = "https://github.com/eigerco/celestia-tendermint-rs.git", rev = "bbe7de8" } +blockstore = { version = "0.1.0", path = "blockstore" } [patch.'https://github.com/eigerco/celestia-tendermint-rs.git'] # Uncomment to apply local changes diff --git a/blockstore/Cargo.toml b/blockstore/Cargo.toml new file mode 100644 index 000000000..b684e2f19 --- /dev/null +++ b/blockstore/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "blockstore" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +async-trait = "0.1.73" +cid = "0.11.0" +dashmap = "5.5.3" +multihash = "0.19.1" +thiserror = "1.0.40" + +[dev-dependencies] +tokio = { version = "1.29.0", features = [ "macros", "rt" ] } + +[package.metadata.docs.rs] +rustdoc-args = ["--cfg", "docs_rs"] diff --git a/blockstore/src/block.rs b/blockstore/src/block.rs new file mode 100644 index 000000000..447b7429f --- /dev/null +++ b/blockstore/src/block.rs @@ -0,0 +1,34 @@ +use cid::CidGeneric; +use thiserror::Error; + +/// Error returned when trying to compute new or parse existing CID. Note that errors here can be +/// specific to particular [`Block`] impl, and don't necessarily indicate invalid CID in general. +#[derive(Debug, Error, PartialEq)] +pub enum CidError { + /// Coded specified in CID is not supported in this context + #[error("Invalid CID codec {0}")] + InvalidCidCodec(u64), + + /// CID's multihash length is different from expected + #[error("Invalid multihash length {0}")] + InvalidMultihashLength(usize), + + /// Encountered multihash code is not supported in this context + #[error("Invalid multihash code {0} expected {1}")] + InvalidMultihashCode(u64, u64), + + /// CID could not be computed for the provided data due to some constraint violation + #[error("Invalid data format {0}")] + InvalidDataFormat(String), + + /// CID is well-formed but contains invalid data + #[error("Invalid CID: {0}")] + InvalidCid(String), +} + +/// Trait for a block of data which can be represented as a string of bytes, which can compute its +/// own CID +pub trait Block: Sync + Send { + fn cid(&self) -> Result, CidError>; + fn data(&self) -> &[u8]; +} diff --git a/blockstore/src/in_memory_blockstore.rs b/blockstore/src/in_memory_blockstore.rs new file mode 100644 index 000000000..03e6c0124 --- /dev/null +++ b/blockstore/src/in_memory_blockstore.rs @@ -0,0 +1,286 @@ +use cid::CidGeneric; +use dashmap::mapref::entry::Entry; +use dashmap::DashMap; +use multihash::Multihash; + +use crate::{Blockstore, BlockstoreError, Result}; + +/// Simple in-memory blockstore implementation. +pub struct InMemoryBlockstore { + map: DashMap, Vec>, +} + +impl InMemoryBlockstore { + /// Create new empty in-memory blockstore + pub fn new() -> Self { + InMemoryBlockstore { + map: DashMap::new(), + } + } + + fn get_cid(&self, cid: &CidGeneric) -> Result>> { + Ok(self.map.get(cid).as_deref().cloned()) + } + + fn insert_cid(&self, cid: CidGeneric, data: &[u8]) -> Result<()> { + let cid_entry = self.map.entry(cid); + if matches!(cid_entry, Entry::Occupied(_)) { + return Err(BlockstoreError::CidExists); + } + + cid_entry.insert(data.to_vec()); + + Ok(()) + } + + fn contains_cid(&self, cid: &CidGeneric) -> bool { + self.map.contains_key(cid) + } +} + +#[cfg_attr(not(docs_rs), async_trait::async_trait)] +impl Blockstore for InMemoryBlockstore { + async fn get(&self, cid: &CidGeneric) -> Result>> { + let hash = cid.hash(); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::new_v1(cid.codec(), hash); + + self.get_cid(&cid) + } + + async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()> { + let hash = cid.hash(); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + let cid = CidGeneric::new_v1(cid.codec(), hash); + + self.insert_cid(cid, data) + } + + async fn has(&self, cid: &CidGeneric) -> Result { + let cid = get_internal_cid(cid)?; + Ok(self.contains_cid(&cid)) + } +} + +impl Default for InMemoryBlockstore { + fn default() -> Self { + Self::new() + } +} + +fn get_internal_cid( + cid: &CidGeneric, +) -> Result> { + let hash = cid.hash(); + let hash = + Multihash::wrap(hash.code(), hash.digest()).map_err(|_| BlockstoreError::CidTooLong)?; + Ok(CidGeneric::new_v1(cid.codec(), hash)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::block::{Block, CidError}; + use std::iter::zip; + use std::result::Result as StdResult; + + #[tokio::test] + async fn test_insert_get() { + let store = InMemoryBlockstore::<128>::new(); + + let cid = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x01, // CID codec = 1 + 0x02, // code = 2 + 0x03, // len = 3 + 1, 2, 3, // hash + ] + .as_ref(), + ) + .unwrap(); + let data = [0xCD; 512]; + + store.put_keyed(&cid, &data).await.unwrap(); + + let retrieved_data = store.get(&cid).await.unwrap().unwrap(); + assert_eq!(data, retrieved_data.as_ref()); + + let another_cid = CidGeneric::<128>::default(); + let missing_block = store.get(&another_cid).await.unwrap(); + assert_eq!(missing_block, None); + } + + #[tokio::test] + async fn test_duplicate_insert() { + let cid0 = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x02, // len = 2 + 0, 0, // hash + ] + .as_ref(), + ) + .unwrap(); + let cid1 = CidGeneric::<128>::read_bytes( + [ + 0x01, // CIDv1 + 0x33, // CID codec + 0x44, // multihash code + 0x02, // len = 2 + 0, 1, // hash + ] + .as_ref(), + ) + .unwrap(); + + let store = InMemoryBlockstore::<128>::new(); + + store.put_keyed(&cid0, &[0x01; 1]).await.unwrap(); + store.put_keyed(&cid1, &[0x02; 1]).await.unwrap(); + let insert_err = store.put_keyed(&cid1, &[0x03; 1]).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidExists); + } + + #[tokio::test] + async fn different_cid_size() { + let cid_bytes = [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x02, // len = 2 + 0, 0, // hash + ]; + let cid0 = CidGeneric::<6>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid1 = CidGeneric::<32>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid2 = CidGeneric::<64>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid3 = CidGeneric::<65>::read_bytes(cid_bytes.as_ref()).unwrap(); + let cid4 = CidGeneric::<128>::read_bytes(cid_bytes.as_ref()).unwrap(); + + let data = [0x99; 5]; + + let store = InMemoryBlockstore::<64>::new(); + store.put_keyed(&cid0, data.as_ref()).await.unwrap(); + + let data0 = store.get(&cid0).await.unwrap().unwrap(); + assert_eq!(data, data0.as_ref()); + let data1 = store.get(&cid1).await.unwrap().unwrap(); + assert_eq!(data, data1.as_ref()); + let data2 = store.get(&cid2).await.unwrap().unwrap(); + assert_eq!(data, data2.as_ref()); + let data3 = store.get(&cid3).await.unwrap().unwrap(); + assert_eq!(data, data3.as_ref()); + let data4 = store.get(&cid4).await.unwrap().unwrap(); + assert_eq!(data, data4.as_ref()); + } + + #[tokio::test] + async fn too_large_cid() { + let cid = CidGeneric::<32>::read_bytes( + [ + 0x01, // CIDv1 + 0x11, // CID codec + 0x22, // multihash code + 0x10, // len = 16 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + ] + .as_ref(), + ) + .unwrap(); + + let store = InMemoryBlockstore::<8>::new(); + let insert_err = store.put_keyed(&cid, [0x00, 1].as_ref()).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidTooLong); + + let insert_err = store.get(&cid).await.unwrap_err(); + assert_eq!(insert_err, BlockstoreError::CidTooLong); + } + + #[tokio::test] + async fn test_block_insert() { + let block = TestBlock([0, 1, 2, 3]); + + let store = InMemoryBlockstore::<8>::new(); + store.put(block).await.unwrap(); + let retrieved_block = store.get(&block.cid().unwrap()).await.unwrap().unwrap(); + assert_eq!(block.data(), &retrieved_block); + } + + #[tokio::test] + async fn test_multiple_blocks_insert() { + let blocks = [ + TestBlock([0, 0, 0, 0]), + TestBlock([0, 0, 0, 1]), + TestBlock([0, 0, 1, 0]), + TestBlock([0, 0, 1, 1]), + TestBlock([0, 1, 0, 0]), + TestBlock([0, 1, 0, 1]), + TestBlock([0, 1, 1, 0]), + TestBlock([0, 1, 1, 1]), + ]; + let uninserted_blocks = [ + TestBlock([1, 0, 0, 0]), + TestBlock([1, 0, 0, 1]), + TestBlock([1, 0, 1, 0]), + TestBlock([1, 1, 0, 1]), + ]; + + let store = InMemoryBlockstore::<8>::new(); + store.put_many(blocks).await.unwrap(); + + for b in blocks { + let cid = b.cid().unwrap(); + assert!(store.has(&cid).await.unwrap()); + let retrieved_block = store.get(&cid).await.unwrap().unwrap(); + assert_eq!(b.data(), &retrieved_block); + } + + for b in uninserted_blocks { + let cid = b.cid().unwrap(); + assert!(!store.has(&cid).await.unwrap()); + assert!(store.get(&cid).await.unwrap().is_none()); + } + } + + #[tokio::test] + async fn test_multiple_keyed() { + let blocks = [[0], [1], [2], [3]]; + let cids = [ + // 4 different arbitrary CIDs + TestBlock([0, 0, 0, 1]).cid().unwrap(), + TestBlock([0, 0, 0, 2]).cid().unwrap(), + TestBlock([0, 0, 0, 3]).cid().unwrap(), + TestBlock([0, 0, 0, 4]).cid().unwrap(), + ]; + let pairs = zip(cids, blocks); + + let store = InMemoryBlockstore::<8>::new(); + store.put_many_keyed(pairs.clone()).await.unwrap(); + + for (cid, block) in pairs { + let retrieved_block = store.get(&cid).await.unwrap().unwrap(); + assert_eq!(block.as_ref(), &retrieved_block); + } + } + + const TEST_CODEC: u64 = 0x0A; + const TEST_MH_CODE: u64 = 0x0A; + + #[derive(Debug, PartialEq, Clone, Copy)] + struct TestBlock(pub [u8; 4]); + + impl Block<8> for TestBlock { + fn cid(&self) -> StdResult, CidError> { + let mh = Multihash::wrap(TEST_MH_CODE, &self.0).unwrap(); + Ok(CidGeneric::new_v1(TEST_CODEC, mh)) + } + + fn data(&self) -> &[u8] { + &self.0 + } + } +} diff --git a/blockstore/src/lib.rs b/blockstore/src/lib.rs new file mode 100644 index 000000000..8fd703716 --- /dev/null +++ b/blockstore/src/lib.rs @@ -0,0 +1,97 @@ +#![cfg_attr(docs_rs, feature(async_fn_in_trait))] + +use cid::CidGeneric; +use thiserror::Error; + +use crate::block::{Block, CidError}; + +pub use crate::in_memory_blockstore::InMemoryBlockstore; + +/// Utilities related to computing CID for the inserted data +pub mod block; +mod in_memory_blockstore; + +/// Error returned when performing operations on [`Blockstore`] +#[derive(Debug, PartialEq, Error)] +pub enum BlockstoreError { + /// Provided CID already exists in blockstore when trying to insert it + #[error("CID already exists in the store")] + CidExists, + + /// Provided CID is longer than max length supported by the blockstore + #[error("CID length longer that max allowed by the store")] + CidTooLong, + + /// Error occured when trying to compute CID with [`HasCid`] trait + /// + /// [`HasCid`]: crate::multihash::HasCid + #[error("Error generating CID: {0}")] + CidError(#[from] CidError), +} + +type Result = std::result::Result; + +/// A IPLD blockstore capable of holding arbitrary data indexed by CID. Implementations can impose +/// limit on supported CID length, and any operations on longer CIDs will fail with [`CidTooLong`] +/// +/// [`CidTooLong`]: BlockstoreError::CidTooLong + +#[cfg_attr(not(docs_rs), async_trait::async_trait)] +pub trait Blockstore { + /// Gets the block from the blockstore + async fn get(&self, cid: &CidGeneric) -> Result>>; + + /// Inserts the data with pre-computed CID. + /// Use [`put`], if you want CID to be computed. + /// + /// [`put`]: Blockstore::put + async fn put_keyed(&self, cid: &CidGeneric, data: &[u8]) -> Result<()>; + + /// Checks whether blockstore has block for provided CID + async fn has(&self, cid: &CidGeneric) -> Result { + Ok(self.get(cid).await?.is_some()) + } + + /// Inserts the data into the blockstore, computing CID using [`HasCid`] trait that needs to be + /// implemented for [`Block`] + /// + /// [`Block`]: crate::multihash::Block + /// [`HasCid`]: crate::multihash::HasCid + async fn put(&self, block: B) -> Result<()> + where + B: Block, + { + let cid = block.cid()?; + self.put_keyed(&cid, block.data()).await + } + + /// Inserts multiple blocks into the blockstore computing their CID + /// If CID computation, or insert itself fails, error is returned and subsequent items are also + /// skipped. + async fn put_many(&self, blocks: I) -> Result<()> + where + B: Block, + I: IntoIterator + Send, + ::IntoIter: Send, + { + for b in blocks { + let cid = b.cid()?; + self.put_keyed(&cid, b.data()).await?; + } + Ok(()) + } + + /// Inserts multiple blocks with pre-computed CID into the blockstore. + /// If any put from the list fails, error is returned and subsequent items are also skipped. + async fn put_many_keyed(&self, blocks: I) -> Result<()> + where + D: AsRef<[u8]> + Send + Sync, + I: IntoIterator, D)> + Send, + ::IntoIter: Send, + { + for (cid, block) in blocks { + self.put_keyed(&cid, block.as_ref()).await?; + } + Ok(()) + } +} diff --git a/types/Cargo.toml b/types/Cargo.toml index bff76db80..462643e31 100644 --- a/types/Cargo.toml +++ b/types/Cargo.toml @@ -5,10 +5,15 @@ edition = "2021" license = "Apache-2.0" [dependencies] +blockstore = { workspace = true } +celestia-proto = { workspace = true } +nmt-rs = { workspace = true } +tendermint = { workspace = true, features = ["std", "rust-crypto"] } +tendermint-proto = { workspace = true } + base64 = "0.21.2" bech32 = "0.9.1" bytes = "1.4.0" -celestia-proto = { workspace = true } cid = { version = "0.11", default-features = false, features = ["std"] } const_format = "0.2.31" ed25519-consensus = { version = "2.1.0", optional = true } @@ -16,14 +21,11 @@ enum_dispatch = "0.3.12" libp2p-identity = { version = "0.2.7", optional = true } multiaddr = { version = "0.18.0", optional = true } multihash = "0.19.1" -nmt-rs = { workspace = true } rand = { version = "0.8.5", optional = true } ruint = { version = "1.8.0", features = ["serde"] } serde = { version = "1.0.164", features = ["derive"] } serde_repr = { version = "0.1", optional = true } sha2 = "0.10.7" -tendermint = { workspace = true, features = ["std", "rust-crypto"] } -tendermint-proto = { workspace = true } thiserror = "1.0.40" [dev-dependencies] diff --git a/types/src/axis.rs b/types/src/axis.rs index e7a810e6a..979131a60 100644 --- a/types/src/axis.rs +++ b/types/src/axis.rs @@ -1,12 +1,12 @@ use std::io::Cursor; use std::result::Result as StdResult; +use blockstore::block::CidError; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use sha2::{Digest, Sha256}; -use crate::multihash::{HasCid, HasMultihash}; use crate::nmt::{NamespacedHashExt, HASH_SIZE}; use crate::DataAvailabilityHeader; use crate::{Error, Result}; @@ -37,7 +37,7 @@ impl TryFrom for AxisType { /// Represents particular particular Column or Row in a specific Data Square, /// paired together with a hash of the axis root. -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct AxisId { pub axis_type: AxisType, pub index: u16, @@ -89,20 +89,21 @@ impl AxisId { bytes.put_u64_le(self.block_height); } - pub(crate) fn decode(buffer: &[u8]) -> Result { + pub(crate) fn decode(buffer: &[u8]) -> Result { if buffer.len() != AXIS_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let mut cursor = Cursor::new(buffer); - let axis_type = cursor.get_u8().try_into()?; + let axis_type = + AxisType::try_from(cursor.get_u8()).map_err(|e| CidError::InvalidCid(e.to_string()))?; let index = cursor.get_u16_le(); let hash = cursor.copy_to_bytes(HASH_SIZE).as_ref().try_into().unwrap(); let block_height = cursor.get_u64_le(); if block_height == 0 { - return Err(Error::ZeroBlockHeight); + return Err(CidError::InvalidCid("Zero block height".to_string())); } Ok(Self { @@ -114,46 +115,44 @@ impl AxisId { } } -impl HasMultihash for AxisId { - fn multihash(&self) -> Result> { - let mut bytes = BytesMut::with_capacity(AXIS_ID_SIZE); - self.encode(&mut bytes); - // length is correct, so unwrap is safe - Ok(Multihash::wrap(AXIS_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for AxisId { - fn codec() -> u64 { - AXIS_ID_CODEC - } -} - impl TryFrom> for AxisId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != AXIS_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, AXIS_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != AXIS_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != AXIS_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode(code, AXIS_ID_MULTIHASH_CODE)); + return Err(CidError::InvalidMultihashCode(code, AXIS_ID_MULTIHASH_CODE)); } AxisId::decode(hash.digest()) } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(axis: AxisId) -> Result { + let mut bytes = BytesMut::with_capacity(AXIS_ID_SIZE); + axis.encode(&mut bytes); + // length is correct, so unwrap is safe + let mh = Multihash::wrap(AXIS_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(AXIS_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -183,7 +182,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let axis_id = AxisId::new(AxisType::Row, 5, &dah, 100).unwrap(); - let cid = axis_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(axis_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), AXIS_ID_MULTIHASH_CODE); @@ -256,7 +255,10 @@ mod tests { assert_eq!(mh.code(), AXIS_ID_MULTIHASH_CODE); assert_eq!(mh.size(), AXIS_ID_SIZE as u8); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!(axis_err, Error::InvalidAxis(0xBE))); + assert_eq!( + axis_err, + CidError::InvalidCid("Invalid axis type: 190".to_string()) + ); } #[test] @@ -280,7 +282,10 @@ mod tests { assert_eq!(mh.code(), AXIS_ID_MULTIHASH_CODE); assert_eq!(mh.size(), AXIS_ID_SIZE as u8); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!(axis_err, Error::ZeroBlockHeight)); + assert_eq!( + axis_err, + CidError::InvalidCid("Zero block height".to_string()) + ); } #[test] @@ -288,10 +293,10 @@ mod tests { let multihash = Multihash::::wrap(999, &[0; AXIS_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(AXIS_ID_CODEC, multihash); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(999, AXIS_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(999, AXIS_ID_MULTIHASH_CODE) + ); } #[test] @@ -300,9 +305,6 @@ mod tests { Multihash::::wrap(AXIS_ID_MULTIHASH_CODE, &[0; AXIS_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(1234, multihash); let axis_err = AxisId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(1234, AXIS_ID_CODEC) - )); + assert_eq!(axis_err, CidError::InvalidCidCodec(1234)); } } diff --git a/types/src/error.rs b/types/src/error.rs index 424f11fd4..a17473c96 100644 --- a/types/src/error.rs +++ b/types/src/error.rs @@ -67,7 +67,7 @@ pub enum Error { #[error("Invalid index of signature in commit {0}, height {1}")] InvalidSignatureIndex(usize, u64), - #[error("Invalid axis: {0}")] + #[error("Invalid axis type: {0}")] InvalidAxis(i32), #[error("Range proof verification failed: {0:?}")] @@ -114,15 +114,6 @@ pub enum Error { #[error("Invalid zero block height")] ZeroBlockHeight, - - #[error("Invalid multihash lenght")] - InvalidMultihashLength(usize), - - #[error("Invalid multihash code {0} expected {1}")] - InvalidMultihashCode(u64, u64), - - #[error("Invalid CID codec {0} expected{1}")] - InvalidCidCodec(u64, u64), } #[derive(Debug, thiserror::Error)] diff --git a/types/src/lib.rs b/types/src/lib.rs index 3b4687a28..d481f3f29 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -8,7 +8,6 @@ mod error; mod extended_header; pub mod fraud_proof; pub mod hash; -pub mod multihash; pub mod namespaced_data; pub mod nmt; #[cfg(feature = "p2p")] diff --git a/types/src/multihash.rs b/types/src/multihash.rs deleted file mode 100644 index b5a785539..000000000 --- a/types/src/multihash.rs +++ /dev/null @@ -1,16 +0,0 @@ -use cid::CidGeneric; -use multihash::Multihash; - -use crate::Result; - -pub trait HasMultihash { - fn multihash(&self) -> Result>; -} - -pub trait HasCid: HasMultihash { - fn cid_v1(&self) -> Result> { - Ok(CidGeneric::::new_v1(Self::codec(), self.multihash()?)) - } - - fn codec() -> u64; -} diff --git a/types/src/namespaced_data.rs b/types/src/namespaced_data.rs index 8305c5860..d6a2fcfe6 100644 --- a/types/src/namespaced_data.rs +++ b/types/src/namespaced_data.rs @@ -1,12 +1,12 @@ use std::io::Cursor; +use blockstore::block::CidError; use bytes::{Buf, BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use sha2::{Digest, Sha256}; use crate::axis::AxisType; -use crate::multihash::{HasCid, HasMultihash}; use crate::nmt::{Namespace, NamespacedHashExt, HASH_SIZE, NS_SIZE}; use crate::{DataAvailabilityHeader, Error, Result}; @@ -15,7 +15,7 @@ pub const NAMESPACED_DATA_ID_MULTIHASH_CODE: u64 = 0x7821; pub const NAMESPACED_DATA_ID_CODEC: u64 = 0x7820; /// Represents shares from a namespace located on a particular row of Data Square -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct NamespacedDataId { pub namespace: Namespace, pub row_index: u16, @@ -67,9 +67,9 @@ impl NamespacedDataId { bytes.put(self.namespace.as_bytes()); } - fn decode(buffer: &[u8]) -> Result { + fn decode(buffer: &[u8]) -> Result { if buffer.len() != NAMESPACED_DATA_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let mut cursor = Cursor::new(buffer); @@ -79,7 +79,7 @@ impl NamespacedDataId { let block_height = cursor.get_u64_le(); if block_height == 0 { - return Err(Error::ZeroBlockHeight); + return Err(CidError::InvalidCid("Zero block height".to_string())); } let namespace = Namespace::from_raw(cursor.copy_to_bytes(NS_SIZE).as_ref()).unwrap(); @@ -93,40 +93,25 @@ impl NamespacedDataId { } } -impl HasMultihash for NamespacedDataId { - fn multihash(&self) -> Result> { - let mut bytes = BytesMut::with_capacity(NAMESPACED_DATA_ID_SIZE); - self.encode(&mut bytes); - // length is correct, so unwrap is safe - Ok(Multihash::wrap(NAMESPACED_DATA_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for NamespacedDataId { - fn codec() -> u64 { - NAMESPACED_DATA_ID_CODEC - } -} - impl TryFrom> for NamespacedDataId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != NAMESPACED_DATA_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, NAMESPACED_DATA_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != NAMESPACED_DATA_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != NAMESPACED_DATA_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode( + return Err(CidError::InvalidMultihashCode( code, NAMESPACED_DATA_ID_MULTIHASH_CODE, )); @@ -136,6 +121,19 @@ impl TryFrom> for NamespacedDataId { } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(namespaced_data_id: NamespacedDataId) -> Result { + let mut bytes = BytesMut::with_capacity(NAMESPACED_DATA_ID_SIZE); + namespaced_data_id.encode(&mut bytes); + // length is correct, so unwrap is safe + let mh = Multihash::wrap(NAMESPACED_DATA_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(NAMESPACED_DATA_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -149,7 +147,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let data_id = NamespacedDataId::new(ns, 5, &dah, 100).unwrap(); - let cid = data_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(data_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), NAMESPACED_DATA_ID_MULTIHASH_CODE); @@ -193,10 +191,10 @@ mod tests { let cid = CidGeneric::::new_v1(NAMESPACED_DATA_ID_CODEC, multihash); let axis_err = NamespacedDataId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(888, NAMESPACED_DATA_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(888, NAMESPACED_DATA_ID_MULTIHASH_CODE) + ); } #[test] @@ -208,9 +206,6 @@ mod tests { .unwrap(); let cid = CidGeneric::::new_v1(4321, multihash); let axis_err = NamespacedDataId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(4321, NAMESPACED_DATA_ID_CODEC) - )); + assert_eq!(axis_err, CidError::InvalidCidCodec(4321)); } } diff --git a/types/src/nmt.rs b/types/src/nmt.rs index eaabf8cd4..417c2cfef 100644 --- a/types/src/nmt.rs +++ b/types/src/nmt.rs @@ -1,4 +1,6 @@ use base64::prelude::*; +use blockstore::block::CidError; +use cid::CidGeneric; use multihash::Multihash; use nmt_rs::simple_merkle::db::MemDb; use nmt_rs::simple_merkle::tree::MerkleHash; @@ -14,7 +16,6 @@ pub use self::namespace_proof::NamespaceProof; pub use self::namespaced_hash::{ NamespacedHashExt, RawNamespacedHash, HASH_SIZE, NAMESPACED_HASH_SIZE, }; -use crate::multihash::{HasCid, HasMultihash}; use crate::{Error, Result}; pub const NS_VER_SIZE: usize = 1; @@ -30,6 +31,8 @@ pub type NamespacedSha2Hasher = nmt_rs::NamespacedSha2Hasher; pub type NamespacedHash = nmt_rs::NamespacedHash; pub type Nmt = nmt_rs::NamespaceMerkleTree, NamespacedSha2Hasher, NS_SIZE>; +pub struct NodePair(NamespacedHash, NamespacedHash); + #[derive(Copy, Clone, Debug, PartialEq, Eq, Ord, PartialOrd, Hash)] pub struct Namespace(nmt_rs::NamespaceId); @@ -196,27 +199,10 @@ impl<'de> Deserialize<'de> for Namespace { } } -impl HasMultihash for (&Namespace, &[u8]) { - fn multihash(&self) -> Result> { - let (ns, data) = self; - let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); - let namespaced_data = [ns.as_bytes(), data].concat(); - - let digest = hasher.hash_leaf(&namespaced_data).to_array(); - // size is correct, so unwrap is safe - Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) - } -} - -impl HasCid for (&Namespace, &[u8]) { - fn codec() -> u64 { - NMT_MULTIHASH_CODE - } -} +impl NodePair { + fn validate_namespace_order(&self) -> Result<()> { + let NodePair(left, right) = self; -impl HasMultihash for (&NamespacedHash, &NamespacedHash) { - fn multihash(&self) -> Result> { - let (left, right) = self; left.validate_namespace_order()?; right.validate_namespace_order()?; @@ -224,17 +210,24 @@ impl HasMultihash for (&NamespacedHash, &NamespacedHash) { return Err(Error::InvalidNmtNodeOrder); } - let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); - - let digest = hasher.hash_nodes(left, right).to_array(); - - Ok(Multihash::wrap(NMT_CODEC, &digest).unwrap()) + Ok(()) } } -impl HasCid for (&NamespacedHash, &NamespacedHash) { - fn codec() -> u64 { - NMT_MULTIHASH_CODE +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(nodes: NodePair) -> Result { + nodes + .validate_namespace_order() + .map_err(|e| CidError::InvalidDataFormat(e.to_string()))?; + + let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); + let digest = hasher.hash_nodes(&nodes.0, &nodes.1).to_array(); + + let mh = Multihash::wrap(NMT_MULTIHASH_CODE, &digest).unwrap(); + + Ok(CidGeneric::new_v1(NMT_CODEC, mh)) } } @@ -467,33 +460,24 @@ mod tests { assert!(matches!(e, Error::UnsupportedNamespaceVersion(254))); } - #[test] - fn test_generate_leaf_multihash() { - let namespace = Namespace::new_v0(&[1, 2, 3]).unwrap(); - let data = [0xCDu8; 512]; - - let hash = (&namespace, data.as_ref()).multihash().unwrap(); - - assert_eq!(hash.code(), NMT_CODEC); - assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); - let hash = NamespacedHash::from_raw(hash.digest()).unwrap(); - assert_eq!(hash.min_namespace(), *namespace); - assert_eq!(hash.max_namespace(), *namespace); - } - #[test] fn test_generate_inner_multihash() { let ns0 = Namespace::new_v0(&[1]).unwrap(); let ns1 = Namespace::new_v0(&[2]).unwrap(); let ns2 = Namespace::new_v0(&[3]).unwrap(); - let left = NamespacedHash::with_min_and_max_ns(*ns0, *ns1); - let right = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); + let nodes = NodePair( + NamespacedHash::with_min_and_max_ns(*ns0, *ns1), + NamespacedHash::with_min_and_max_ns(*ns1, *ns2), + ); - let hash = (&left, &right).multihash().unwrap(); + let cid = CidGeneric::try_from(nodes).unwrap(); + assert_eq!(cid.codec(), NMT_CODEC); - assert_eq!(hash.code(), NMT_CODEC); + let hash = cid.hash(); + assert_eq!(hash.code(), NMT_MULTIHASH_CODE); assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); + let hash = NamespacedHash::from_raw(hash.digest()).unwrap(); assert_eq!(hash.min_namespace(), *ns0); assert_eq!(hash.max_namespace(), *ns2); @@ -505,13 +489,16 @@ mod tests { let ns1 = Namespace::new_v0(&[2]).unwrap(); let ns2 = Namespace::new_v0(&[3]).unwrap(); - let left = NamespacedHash::with_min_and_max_ns(*ns1, *ns2); - let right = NamespacedHash::with_min_and_max_ns(*ns0, *ns0); + let nodes = NodePair( + NamespacedHash::with_min_and_max_ns(*ns1, *ns2), + NamespacedHash::with_min_and_max_ns(*ns0, *ns0), + ); + let result = CidGeneric::try_from(nodes).unwrap_err(); - //let multihash = Code::Sha256NamespaceFlagged; - //let digest_result = multihash.digest_nodes(&left, &right).unwrap_err(); - let result = (&left, &right).multihash().unwrap_err(); - assert!(matches!(result, Error::InvalidNmtNodeOrder)); + assert_eq!( + result, + CidError::InvalidDataFormat("Invalid nmt node order".to_string()) + ); } #[test] diff --git a/types/src/sample.rs b/types/src/sample.rs index 812e251fe..fd417938b 100644 --- a/types/src/sample.rs +++ b/types/src/sample.rs @@ -1,11 +1,11 @@ use std::mem::size_of; +use blockstore::block::CidError; use bytes::{BufMut, BytesMut}; use cid::CidGeneric; use multihash::Multihash; use crate::axis::{AxisId, AxisType}; -use crate::multihash::{HasCid, HasMultihash}; use crate::DataAvailabilityHeader; use crate::{Error, Result}; @@ -14,7 +14,7 @@ pub const SAMPLE_ID_MULTIHASH_CODE: u64 = 0x7801; pub const SAMPLE_ID_CODEC: u64 = 0x7800; /// Represents particular sample along the axis on specific Data Square -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Clone, Copy)] pub struct SampleId { pub axis: AxisId, pub index: u16, @@ -57,9 +57,9 @@ impl SampleId { bytes.put_u16_le(self.index); } - fn decode(buffer: &[u8]) -> Result { + fn decode(buffer: &[u8]) -> Result { if buffer.len() != SAMPLE_ID_SIZE { - return Err(Error::InvalidMultihashLength(buffer.len())); + return Err(CidError::InvalidMultihashLength(buffer.len())); } let (axis_id, index) = buffer.split_at(AxisId::size()); @@ -71,47 +71,48 @@ impl SampleId { } } -impl HasMultihash for SampleId { - fn multihash(&self) -> Result> { - let mut bytes = BytesMut::with_capacity(Self::size()); - - self.encode(&mut bytes); - - Ok(Multihash::::wrap(SAMPLE_ID_MULTIHASH_CODE, &bytes[..]).unwrap()) - } -} - -impl HasCid for SampleId { - fn codec() -> u64 { - SAMPLE_ID_CODEC - } -} - impl TryFrom> for SampleId { - type Error = Error; + type Error = CidError; fn try_from(cid: CidGeneric) -> Result { let codec = cid.codec(); if codec != SAMPLE_ID_CODEC { - return Err(Error::InvalidCidCodec(codec, SAMPLE_ID_CODEC)); + return Err(CidError::InvalidCidCodec(codec)); } let hash = cid.hash(); let size = hash.size() as usize; if size != SAMPLE_ID_SIZE { - return Err(Error::InvalidMultihashLength(size)); + return Err(CidError::InvalidMultihashLength(size)); } let code = hash.code(); if code != SAMPLE_ID_MULTIHASH_CODE { - return Err(Error::InvalidMultihashCode(code, SAMPLE_ID_MULTIHASH_CODE)); + return Err(CidError::InvalidMultihashCode( + code, + SAMPLE_ID_MULTIHASH_CODE, + )); } SampleId::decode(hash.digest()) } } +impl TryFrom for CidGeneric { + type Error = CidError; + + fn try_from(sample_id: SampleId) -> Result { + let mut bytes = BytesMut::with_capacity(SAMPLE_ID_SIZE); + // length is correct, so unwrap is safe + sample_id.encode(&mut bytes); + + let mh = Multihash::wrap(SAMPLE_ID_MULTIHASH_CODE, &bytes[..]).unwrap(); + + Ok(CidGeneric::new_v1(SAMPLE_ID_CODEC, mh)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -124,7 +125,7 @@ mod tests { column_roots: vec![NamespacedHash::empty_root(); 10], }; let sample_id = SampleId::new(AxisType::Row, 5, &dah, 100).unwrap(); - let cid = sample_id.cid_v1().unwrap(); + let cid = CidGeneric::try_from(sample_id).unwrap(); let multihash = cid.hash(); assert_eq!(multihash.code(), SAMPLE_ID_MULTIHASH_CODE); @@ -183,10 +184,10 @@ mod tests { let multihash = Multihash::::wrap(888, &[0; SAMPLE_ID_SIZE]).unwrap(); let cid = CidGeneric::::new_v1(SAMPLE_ID_CODEC, multihash); let axis_err = SampleId::try_from(cid).unwrap_err(); - assert!(matches!( + assert_eq!( axis_err, - Error::InvalidMultihashCode(888, SAMPLE_ID_MULTIHASH_CODE) - )); + CidError::InvalidMultihashCode(888, SAMPLE_ID_MULTIHASH_CODE) + ); } #[test] @@ -196,9 +197,6 @@ mod tests { .unwrap(); let cid = CidGeneric::::new_v1(4321, multihash); let axis_err = SampleId::try_from(cid).unwrap_err(); - assert!(matches!( - axis_err, - Error::InvalidCidCodec(4321, SAMPLE_ID_CODEC) - )); + assert!(matches!(axis_err, CidError::InvalidCidCodec(4321))); } } diff --git a/types/src/share.rs b/types/src/share.rs index 261b4d51d..905576915 100644 --- a/types/src/share.rs +++ b/types/src/share.rs @@ -1,9 +1,17 @@ +use blockstore::block::{Block, CidError}; use celestia_proto::share::p2p::shrex::nd::NamespaceRowResponse as RawNamespacedRow; +use cid::CidGeneric; +use multihash::Multihash; +use nmt_rs::simple_merkle::tree::MerkleHash; +use nmt_rs::NamespaceMerkleHasher; use serde::{Deserialize, Serialize}; use tendermint_proto::Protobuf; use crate::consts::appconsts; -use crate::nmt::{Namespace, NamespaceProof, NS_SIZE}; +use crate::nmt::{ + Namespace, NamespaceProof, NamespacedSha2Hasher, NMT_CODEC, NMT_ID_SIZE, NMT_MULTIHASH_CODE, + NS_SIZE, +}; use crate::{Error, Result}; mod info_byte; @@ -70,6 +78,22 @@ impl AsRef<[u8]> for Share { } } +impl Block for Share { + fn cid(&self) -> Result, CidError> { + let hasher = NamespacedSha2Hasher::with_ignore_max_ns(true); + let digest = hasher.hash_leaf(self.as_ref()).iter().collect::>(); + + // size is correct, so unwrap is safe + let mh = Multihash::wrap(NMT_MULTIHASH_CODE, &digest).unwrap(); + + Ok(CidGeneric::new_v1(NMT_CODEC, mh)) + } + + fn data(&self) -> &[u8] { + &self.data + } +} + #[derive(Serialize, Deserialize)] #[serde(transparent)] struct RawNamespacedShares { @@ -157,6 +181,7 @@ impl From for RawNamespacedRow { #[cfg(test)] mod tests { use super::*; + use crate::nmt::{NamespacedHash, NAMESPACED_HASH_SIZE}; use base64::prelude::*; #[cfg(target_arch = "wasm32")] @@ -330,4 +355,21 @@ mod tests { assert_eq!(ns_shares.rows[0].shares.len(), 1); assert!(!ns_shares.rows[0].proof.is_of_absence()); } + + #[test] + fn test_generate_leaf_multihash() { + let namespace = Namespace::new_v0(&[1, 2, 3]).unwrap(); + let mut data = [0xCDu8; appconsts::SHARE_SIZE]; + data[..NS_SIZE].copy_from_slice(namespace.as_bytes()); + let share = Share::from_raw(&data).unwrap(); + + let cid = share.cid().unwrap(); + assert_eq!(cid.codec(), NMT_CODEC); + let hash = cid.hash(); + assert_eq!(hash.code(), NMT_MULTIHASH_CODE); + assert_eq!(hash.size(), NAMESPACED_HASH_SIZE as u8); + let hash = NamespacedHash::try_from(hash.digest()).unwrap(); + assert_eq!(hash.min_namespace(), *namespace); + assert_eq!(hash.max_namespace(), *namespace); + } }