From ece7b89d039f0846a667ea3e4ba58aadbc174ec5 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 7 Feb 2024 16:19:19 +0100 Subject: [PATCH 01/43] first implementation --- Cargo.lock | 37 +- crates/node/Cargo.toml | 1 + crates/node/src/asset_manager/mod.rs | 233 +++++----- crates/node/src/cli.rs | 15 + .../download_manager.rs | 51 ++- crates/node/src/event_loop/mod.rs | 402 ++++++++++++++++++ crates/node/src/main.rs | 188 ++++---- crates/node/src/mempool/mod.rs | 40 +- crates/node/src/networking/mod.rs | 1 - crates/node/src/networking/p2p/pea2pea.rs | 62 ++- crates/node/src/rpc_server/mod.rs | 67 +-- crates/node/src/scheduler/mod.rs | 64 ++- crates/node/src/storage/database/postgres.rs | 33 +- crates/node/src/storage/file.rs | 2 +- crates/node/src/types/file.rs | 71 ++++ crates/node/src/types/mod.rs | 3 +- crates/node/src/types/task.rs | 20 +- crates/node/src/types/transaction.rs | 89 +++- crates/node/src/workflow/mod.rs | 68 +-- 19 files changed, 1011 insertions(+), 436 deletions(-) rename crates/node/src/{networking => event_loop}/download_manager.rs (79%) create mode 100644 crates/node/src/event_loop/mod.rs create mode 100644 crates/node/src/types/file.rs diff --git a/Cargo.lock b/Cargo.lock index 7e343722..51af7db4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -963,9 +963,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", @@ -978,9 +978,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -988,15 +988,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" dependencies = [ "futures-core", "futures-task", @@ -1016,15 +1016,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-macro" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -1033,15 +1033,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -1055,9 +1055,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -1159,6 +1159,7 @@ dependencies = [ "console-subscriber", "ecies", "eyre", + "futures", "futures-util", "hex", "home", diff --git a/crates/node/Cargo.toml b/crates/node/Cargo.toml index d389de3a..c3ab26ff 100644 --- a/crates/node/Cargo.toml +++ b/crates/node/Cargo.toml @@ -17,6 +17,7 @@ clap = { version = "4", features = ["derive", "env", "string"] } console-subscriber = "0.2" ecies = {version = "0.2", default-features = false, features = ["pure"]} eyre = "0.6.8" +futures = "0.3.30" futures-util = "0.3" hex = "0.4" home = "0.5" diff --git a/crates/node/src/asset_manager/mod.rs b/crates/node/src/asset_manager/mod.rs index ec97ec14..3653b57a 100644 --- a/crates/node/src/asset_manager/mod.rs +++ b/crates/node/src/asset_manager/mod.rs @@ -1,3 +1,4 @@ +//TODO to be removed use crate::{ cli::Config, storage::Database, @@ -32,20 +33,20 @@ enum AssetManagerError { /// VM images for those programs. Similarly, Run transaction has associated /// input data which must be downloaded, but also built into workspace volume /// for execution. -pub struct AssetManager { +pub struct AssetManagerOld { config: Arc, database: Arc, http_client: reqwest::Client, http_peer_list: Arc>>>, } -impl AssetManager { +impl AssetManagerOld { pub fn new( config: Arc, database: Arc, http_peer_list: Arc>>>, ) -> Self { - AssetManager { + Self { config, database, http_client: reqwest::Client::new(), @@ -58,14 +59,14 @@ impl AssetManager { loop { for tx_hash in self.database.get_incomplete_assets().await? { if let Some(tx) = self.database.find_transaction(&tx_hash).await? { - if let Err(err) = self.process_transaction(&tx).await { - tracing::error!( - "failed to process transaction (hash: {}) assets: {}", - tx.hash, - err - ); - continue; - } + // if let Err(err) = self.process_transaction(&tx).await { + // tracing::error!( + // "failed to process transaction (hash: {}) assets: {}", + // tx.hash, + // err + // ); + // continue; + // } self.database.mark_asset_complete(&tx_hash).await?; } else { @@ -87,109 +88,109 @@ impl AssetManager { self.database.add_asset(&tx.hash).await } - async fn process_transaction(&self, tx: &Transaction) -> Result<()> { - match tx.payload { - transaction::Payload::Deploy { .. } => self.process_deployment(tx).await, - transaction::Payload::Run { .. } => self.process_run(tx).await, - // Other transaction types don't have external assets that would - // need processing. - _ => Ok(()), - } - } - - async fn process_deployment(&self, tx: &Transaction) -> Result<()> { - let (prover, verifier) = match tx.payload.clone() { - Payload::Deploy { - name: _, - prover, - verifier, - } => (Program::from(prover), Program::from(verifier)), - _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), - }; - - self.process_program(&prover).await?; - self.process_program(&verifier).await?; - - Ok(()) - } - - async fn process_program(&self, program: &Program) -> Result<()> { - self.download_image(program).await - } - - async fn process_run(&self, tx: &Transaction) -> Result<()> { - let workflow = match tx.payload.clone() { - Payload::Run { workflow } => workflow, - _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), - }; - - // TODO: Ideally the following would happen concurrently for each file... - for step in workflow.steps { - for input in step.inputs { - match input { - ProgramData::Input { - file_name, - file_url, - checksum, - } => { - let f = types::File { - tx: tx.hash, - name: file_name, - url: file_url, - }; - crate::networking::download_manager::download_file( - &f.url, - &self.config.data_directory, - f.get_file_relative_path() - .to_str() - .ok_or(eyre!("Download bad file path: {:?}", f.name))?, - self.get_peer_list().await, - &self.http_client, - checksum.into(), - ) - .await?; - } - ProgramData::Output { .. } => { - /* ProgramData::Output asinput means it comes from another - program execution -> skip this branch. */ - } - } - } - } - - Ok(()) - } - - /// download downloads file from the given `url` and saves it to file in `file_path`. - async fn download_image(&self, program: &Program) -> Result<()> { - let file_path = PathBuf::new() - .join("images") - .join(program.hash.to_string()) - .join(&program.image_file_name); - tracing::info!( - "asset download url:{} file_path:{file_path:?} file_checksum:{}", - program.image_file_url, - program.image_file_checksum - ); - crate::networking::download_manager::download_file( - &program.image_file_url, - &self.config.data_directory, - file_path - .to_str() - .ok_or(eyre!("Download bad file path: {:?}", file_path))?, - self.get_peer_list().await, - &self.http_client, - (&*program.image_file_checksum).into(), - ) - .await - } - - async fn get_peer_list(&self) -> Vec<(SocketAddr, Option)> { - self.http_peer_list - .read() - .await - .iter() - .map(|(a, p)| (*a, *p)) - .collect() - } + // async fn process_transaction(&self, tx: &Transaction) -> Result<()> { + // match tx.payload { + // transaction::Payload::Deploy { .. } => self.process_deployment(tx).await, + // transaction::Payload::Run { .. } => self.process_run(tx).await, + // // Other transaction types don't have external assets that would + // // need processing. + // _ => Ok(()), + // } + // } + + // async fn process_deployment(&self, tx: &Transaction) -> Result<()> { + // let (prover, verifier) = match tx.payload.clone() { + // Payload::Deploy { + // name: _, + // prover, + // verifier, + // } => (Program::from(prover), Program::from(verifier)), + // _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), + // }; + + // self.process_program(&prover).await?; + // self.process_program(&verifier).await?; + + // Ok(()) + // } + + // async fn process_program(&self, program: &Program) -> Result<()> { + // self.download_image(program).await + // } + + // async fn process_run(&self, tx: &Transaction) -> Result<()> { + // let workflow = match tx.payload.clone() { + // Payload::Run { workflow } => workflow, + // _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), + // }; + + // // TODO: Ideally the following would happen concurrently for each file... + // for step in workflow.steps { + // for input in step.inputs { + // match input { + // ProgramData::Input { + // file_name, + // file_url, + // checksum, + // } => { + // let f = types::File { + // tx: tx.hash, + // name: file_name, + // url: file_url, + // }; + // crate::networking::download_manager::download_file( + // &f.url, + // &self.config.data_directory, + // f.get_file_relative_path() + // .to_str() + // .ok_or(eyre!("Download bad file path: {:?}", f.name))?, + // self.get_peer_list().await, + // &self.http_client, + // checksum.into(), + // ) + // .await?; + // } + // ProgramData::Output { .. } => { + // /* ProgramData::Output asinput means it comes from another + // program execution -> skip this branch. */ + // } + // } + // } + // } + + // Ok(()) + // } + + // /// download downloads file from the given `url` and saves it to file in `file_path`. + // async fn download_image(&self, program: &Program) -> Result<()> { + // let file_path = PathBuf::new() + // .join("images") + // .join(program.hash.to_string()) + // .join(&program.image_file_name); + // tracing::info!( + // "asset download url:{} file_path:{file_path:?} file_checksum:{}", + // program.image_file_url, + // program.image_file_checksum + // ); + // crate::networking::download_manager::download_file( + // &program.image_file_url, + // &self.config.data_directory, + // file_path + // .to_str() + // .ok_or(eyre!("Download bad file path: {:?}", file_path))?, + // self.get_peer_list().await, + // &self.http_client, + // (&*program.image_file_checksum).into(), + // ) + // .await + // } + + // async fn get_peer_list(&self) -> Vec<(SocketAddr, Option)> { + // self.http_peer_list + // .read() + // .await + // .iter() + // .map(|(a, p)| (*a, *p)) + // .collect() + // } } diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 486a6e42..323901da 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -148,6 +148,13 @@ pub enum PeerCommand { #[derive(Debug, Args)] pub struct P2PBeaconConfig { + #[arg( + long, + long_help = "Directory where the node should store its data", + env = "GEVULOT_DATA_DIRECTORY", + default_value_os_t = PathBuf::from("/var/lib/gevulot"), + )] + pub data_directory: PathBuf, #[arg( long, long_help = "P2P listen address", @@ -170,6 +177,14 @@ pub struct P2PBeaconConfig { default_value = "Pack my box with five dozen liquor jugs." )] pub p2p_psk_passphrase: String, + + #[arg( + long, + long_help = "Port open to download transaction data between nodes. Use P2P interface to bind.", + env = "GEVULOT_HTTP_PORT", + default_value = "9995" + )] + pub http_download_port: u16, } #[derive(Debug, Subcommand)] diff --git a/crates/node/src/networking/download_manager.rs b/crates/node/src/event_loop/download_manager.rs similarity index 79% rename from crates/node/src/networking/download_manager.rs rename to crates/node/src/event_loop/download_manager.rs index 387e6410..9ad2dde0 100644 --- a/crates/node/src/networking/download_manager.rs +++ b/crates/node/src/event_loop/download_manager.rs @@ -1,4 +1,4 @@ -use crate::cli::Config; +use crate::types::file::AssetFile; use eyre::eyre; use eyre::Result; use futures_util::TryStreamExt; @@ -11,6 +11,8 @@ use hyper::{Request, Response, StatusCode}; use hyper_util::rt::TokioIo; use std::net::SocketAddr; use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; @@ -18,17 +20,19 @@ use tokio::task::JoinHandle; use tokio_util::io::ReaderStream; /// download_file downloads file from the given `url` and saves it to file in `local_directory_path` + / + `file`. -pub async fn download_file( - url: &str, +pub async fn download_asset_file( + // url: &str, local_directory_path: &Path, - file: &str, - http_peer_list: Vec<(SocketAddr, Option)>, + // file: &str, + http_peer_list: &[(SocketAddr, Option)], http_client: &reqwest::Client, - file_hash: gevulot_node::types::Hash, + asset_file: AssetFile, + asset_checksum: gevulot_node::types::Hash, + // file_hash: gevulot_node::types::Hash, ) -> Result<()> { - tracing::trace!("download_file url:{url} local_directory_path:{local_directory_path:?} file:{file} file_hash:{file_hash} http_peer_list:{http_peer_list:?}"); - let url = reqwest::Url::parse(url)?; - let mut resp = match http_client.get(url.clone()).send().await { + let local_relative_file_path = asset_file.get_relatif_path(); + tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); + let mut resp = match http_client.get(asset_file.url).send().await { Ok(resp) => resp, Err(err) => { let peer_urls: Vec = http_peer_list @@ -39,7 +43,7 @@ pub async fn download_file( let mut url = reqwest::Url::parse("http://localhost").unwrap(); //unwrap always succeed url.set_ip_host(peer.ip()).unwrap(); //unwrap always succeed url.set_port(Some(port)).unwrap(); //unwrap always succeed - url.set_path(file); //unwrap always succeed + url.set_path(&local_relative_file_path.to_str().unwrap()); //unwrap Path alway ok url }) }) @@ -53,13 +57,14 @@ pub async fn download_file( } } resp.ok_or(eyre!( - "Download no host found to download the file: {file:?}" + "Download no host found to download the file: {}", + asset_file.name ))? } }; if resp.status() == reqwest::StatusCode::OK { - let file_path = local_directory_path.join(file); + let file_path = local_directory_path.join(&local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { if let Ok(false) = tokio::fs::try_exists(parent).await { @@ -84,17 +89,17 @@ pub async fn download_file( } fd.flush().await?; - let checksum: gevulot_node::types::Hash = (&hasher.finalize()).into(); - if checksum != file_hash { - Err(eyre!("Download file: {:?}, bad checksum", file)) + let checksum: crate::types::Hash = (&hasher.finalize()).into(); + if checksum != asset_checksum { + Err(eyre!("Download file: {:?}, bad checksum", asset_file.name)) } else { //rename to original name Ok(std::fs::rename(tmp_file_path, file_path)?) } } else { Err(eyre!( - "failed to download file from {}: response status: {}", - url, + "failed to download file: {:?} response status: {}", + asset_file.name, resp.status() )) } @@ -102,13 +107,17 @@ pub async fn download_file( //start the local server and serve the specified file path. //Return the server task join handle. -pub async fn serve_files(config: &Config) -> Result> { - let mut bind_addr = config.p2p_listen_addr; - bind_addr.set_port(config.http_download_port); +pub async fn serve_files( + mut bind_addr: SocketAddr, + http_download_port: u16, + data_directory: Arc, +) -> Result> { + // let mut bind_addr = config.p2p_listen_addr; + bind_addr.set_port(http_download_port); let listener = TcpListener::bind(bind_addr).await?; let jh = tokio::spawn({ - let data_directory = config.data_directory.clone(); + let data_directory = data_directory.clone(); async move { tracing::info!( "listening for http at {}", diff --git a/crates/node/src/event_loop/mod.rs b/crates/node/src/event_loop/mod.rs new file mode 100644 index 00000000..9e3dd869 --- /dev/null +++ b/crates/node/src/event_loop/mod.rs @@ -0,0 +1,402 @@ +use crate::types::transaction::Payload; +use crate::types::transaction::TransactionError; +use crate::types::Hash; +use crate::types::Signature; +use crate::types::Transaction; +use crate::Mempool; +use futures::future::join_all; +use futures_util::stream::FuturesUnordered; +use futures_util::Stream; +use futures_util::StreamExt; +use futures_util::TryFutureExt; +use gevulot_node::types::transaction::AclWhitelist; +use libsecp256k1::verify; +use libsecp256k1::Message; +use libsecp256k1::PublicKey; +use sha3::{Digest, Sha3_256}; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use thiserror::Error; +use tokio::sync::mpsc; +use tokio::sync::mpsc::UnboundedReceiver; +use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::oneshot; +use tokio::sync::RwLock; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::UnboundedReceiverStream; + +mod download_manager; + +#[derive(Error, Debug)] +pub enum EventProcessError { + #[error("Fail to rcv Tx from the channel: {0}")] + RcvChannelError(#[from] tokio::sync::oneshot::error::RecvError), + #[error("Fail to send the Tx on the channel: {0}")] + SendChannelError(#[from] tokio::sync::mpsc::error::SendError<(RcvTx, Option)>), + #[error("Fail to send the Tx on the channel: {0}")] + PropagateTxError(#[from] tokio::sync::mpsc::error::SendError), + #[error("validation fail: {0}")] + ValidateError(String), + #[error("Tx asset fail to download because {0}")] + DownloadAssetError(String), + #[error("Save Tx error: {0}")] + SaveTxError(String), +} + +pub type CallbackSender = oneshot::Sender>; + +///Content of all Tx. +#[derive(Debug, Clone)] +struct EventTxContent { + pub author: PublicKey, + pub hash: Hash, + pub payload: Payload, + pub nonce: u64, + pub signature: Signature, +} + +impl From for EventTxContent { + fn from(tx: Transaction) -> Self { + EventTxContent { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + } + } +} + +impl From for Transaction { + fn from(content: EventTxContent) -> Self { + Transaction { + author: content.author, + hash: content.hash, + payload: content.payload, + nonce: content.nonce, + signature: content.signature, + propagated: true, + executed: false, + } + } +} + +//event base type. +#[derive(Debug, Clone)] +pub struct EventTx { + content: EventTxContent, + tx_type: T, +} + +impl EventTx { + fn transition_new(&self, tx_type: N) -> EventTx { + EventTx { + content: self.content.clone(), + tx_type, + } + } + fn transition(self, tx_type: N) -> EventTx { + EventTx { + content: self.content, + tx_type, + } + } +} + +#[derive(Debug)] +enum SourceTxType { + P2P, + RPC, + TXRESULT, +} + +impl EventTx { + async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { + let tx: Transaction = self.content.clone().into(); + tx.validate()?; + + // Secondly verify that author is whitelisted. + if !acl_whitelist.contains(&tx.author).await? { + return Err(TransactionError::Validation( + "Tx permission denied signer not authorized".to_string(), + )); + } + + if let Payload::Run { ref workflow } = self.content.payload { + let mut programs = HashSet::new(); + for step in &workflow.steps { + if !programs.insert(step.program) { + return Err(TransactionError::Validation(format!( + "multiple programs in workflow: {}", + &step.program + ))); + } + } + } + + Ok(()) + } + + fn verify_tx_signature(&self) -> bool { + let mut hasher = Sha3_256::new(); + let mut buf = vec![]; + hasher.update(self.content.author.serialize()); + self.content.payload.serialize_into(&mut buf); + hasher.update(buf); + hasher.update(self.content.nonce.to_be_bytes()); + + let hash: Hash = (&hasher.finalize()[0..32]).into(); + let msg: Message = hash.into(); + verify(&msg, &self.content.signature.into(), &self.content.author) + } + + fn propagate(self) -> Option { + match self.tx_type { + SourceTxType::P2P => None, + SourceTxType::RPC => Some(PropagateTx(EventTx { + content: self.content, + tx_type: ConsistentTx, + })), + SourceTxType::TXRESULT => Some(PropagateTx(EventTx { + content: self.content, + tx_type: ConsistentTx, + })), + } + } +} + +#[derive(Debug)] +struct ConsistentTx; + +//event list +pub struct RcvTx(EventTx); +impl RcvTx { + async fn process_event( + self, + acl_whitelist: &impl AclWhitelist, + ) -> Result { + match self.0.validate_tx(acl_whitelist).await { + Ok(()) => Ok(DownloadTx(self.0)), + Err(err) => Err(EventProcessError::ValidateError(format!( + "Tx validation fail:{err}" + ))), + } + } +} +struct DownloadTx(EventTx); +impl DownloadTx { + async fn process_event( + self, + local_directory_path: &PathBuf, + http_peer_list: Vec<(SocketAddr, Option)>, + ) -> Result<(NewTx, Option), EventProcessError> { + let tx_hash = self.0.content.hash; + let http_client = reqwest::Client::new(); + let asset_file_list = self + .0 + .content + .payload + .get_asset_list(tx_hash) + .map_err(|err| { + EventProcessError::DownloadAssetError(format!( + "Asset file param conversion error:{err}" + )) + })?; + + let futures: Vec<_> = asset_file_list + .into_iter() + .map(|(asset_file, checksum)| { + download_manager::download_asset_file( + local_directory_path, + &http_peer_list, + &http_client, + asset_file, + checksum, + ) + }) + .collect(); + join_all(futures) + .await + .into_iter() + .map(|res| res) + .collect::, _>>() + .map_err(|err| { + EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) + })?; + let newtx = NewTx(self.0.transition_new(ConsistentTx)); + let propagate = self.0.propagate(); + Ok((newtx, propagate)) + } +} +struct PropagateTx(EventTx); +impl PropagateTx { + async fn process_event( + self, + p2p_sender: &UnboundedSender, + ) -> Result<(), EventProcessError> { + let tx: Transaction = self.0.content.into(); + p2p_sender.send(tx).map_err(|err| err.into()) + } +} + +struct NewTx(EventTx); +impl NewTx { + async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { + let tx: Transaction = self.0.content.into(); + mempool + .add(tx) + .map_err(|err| EventProcessError::SaveTxError(format!("{err}"))) + .await + } +} + +//send Tx interface with outside the module +pub struct RpcSender; +#[derive(Clone)] +pub struct P2pSender; +pub struct TxResultSender; + +#[derive(Debug, Clone)] +pub struct TxEventSender { + sender: UnboundedSender<(RcvTx, Option)>, + _marker: PhantomData, +} + +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::P2P, + }; + self.sender + .send((RcvTx(event), None)) + .map_err(|err| err.into()) + } +} + +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let (sender, rx) = oneshot::channel(); + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::RPC, + }; + self.sender + .send((RcvTx(event), Some(sender))) + .map_err(|err| EventProcessError::from(err))?; + rx.await? + } +} + +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let (sender, rx) = oneshot::channel(); + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::TXRESULT, + }; + self.sender + .send((RcvTx(event), Some(sender))) + .map_err(|err| EventProcessError::from(err))?; + rx.await? + } +} + +pub async fn start_event_loop( + local_directory_path: PathBuf, + bind_addr: SocketAddr, + http_download_port: u16, + http_peer_list: Arc>>>, + acl_whitelist: Arc, + mempool: Arc>, +) -> eyre::Result<( + JoinHandle<()>, + UnboundedSender<(RcvTx, Option)>, + impl Stream, +)> { + let local_directory_path = Arc::new(local_directory_path); + //start http download manager + let download_jh = + download_manager::serve_files(bind_addr, http_download_port, local_directory_path.clone()) + .await?; + + let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel::<(RcvTx, Option)>(); + + let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::(); + let p2p_stream = UnboundedReceiverStream::new(p2p_recv); + let jh = tokio::spawn({ + let local_directory_path = local_directory_path.clone(); + + async move { + while let Some((event, callback)) = rcv_tx_event_rx.recv().await { + //process RcvTx(EventTx) event + let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; + tokio::spawn({ + let p2p_sender = p2p_sender.clone(); + let local_directory_path = local_directory_path.clone(); + let acl_whitelist = acl_whitelist.clone(); + let mempool = mempool.clone(); + async move { + let res = event + .process_event(acl_whitelist.as_ref()) + .and_then(|download_event| { + download_event.process_event(&local_directory_path, http_peer_list) + }) + .and_then(|(new_tx, propagate_tx)| async move { + if let Some(propagate_tx) = propagate_tx { + propagate_tx.process_event(&p2p_sender).await?; + } + new_tx.process_event(&mut *(mempool.write().await)).await?; + + Ok(()) + }) + .await; + //send the execution result back if needed. + if let Some(callback) = callback { + //forget the result because if the RPC connection is closed the send can fail. + let _ = callback.send(res); + } + } + }); + } + } + }); + Ok((jh, tx, p2p_stream)) +} + +async fn convert_peer_list_to_vec( + http_peer_list: &tokio::sync::RwLock>>, +) -> Vec<(SocketAddr, Option)> { + http_peer_list + .read() + .await + .iter() + .map(|(a, p)| (*a, *p)) + .collect() +} diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 551276c4..8c87426e 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -1,16 +1,10 @@ #![allow(dead_code)] #![allow(unused_variables)] -use std::{ - io::{ErrorKind, Write}, - net::ToSocketAddrs, - path::PathBuf, - sync::{Arc, Mutex}, - thread::sleep, - time::Duration, -}; - -use asset_manager::AssetManager; +use crate::event_loop::start_event_loop; +use crate::event_loop::P2pSender; +use crate::event_loop::RpcSender; +use crate::event_loop::TxEventSender; use async_trait::async_trait; use clap::Parser; use cli::{ @@ -23,7 +17,19 @@ use libsecp256k1::{PublicKey, SecretKey}; use pea2pea::Pea2Pea; use rand::{rngs::StdRng, SeedableRng}; use sqlx::postgres::PgPoolOptions; +use std::collections::HashMap; +use std::net::SocketAddr; +use std::{ + io::{ErrorKind, Write}, + net::ToSocketAddrs, + path::PathBuf, + sync::{Arc, Mutex}, + thread::sleep, + time::Duration, +}; +use tokio::sync::mpsc; use tokio::sync::{Mutex as TMutex, RwLock}; +use tokio_stream::wrappers::UnboundedReceiverStream; use tonic::transport::Server; use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan, EnvFilter}; use types::{Hash, Transaction}; @@ -31,6 +37,7 @@ use workflow::WorkflowEngine; mod asset_manager; mod cli; +mod event_loop; mod mempool; mod nanos; mod networking; @@ -138,7 +145,10 @@ impl mempool::Storage for storage::Database { } async fn set(&self, tx: &Transaction) -> Result<()> { - self.add_transaction(tx).await + let tx_hash = tx.hash; + self.add_transaction(tx).await?; + self.add_asset(&tx_hash).await?; + self.mark_asset_complete(&tx_hash).await } async fn fill_deque(&self, deque: &mut std::collections::VecDeque) -> Result<()> { @@ -157,48 +167,56 @@ impl workflow::TransactionStore for storage::Database { } } -struct P2PTxHandler { - mempool: Arc>, - database: Arc, -} - -impl P2PTxHandler { - pub fn new(mempool: Arc>, database: Arc) -> Self { - Self { mempool, database } - } -} - -#[async_trait::async_trait] -impl networking::p2p::TxHandler for P2PTxHandler { - async fn recv_tx(&self, tx: Transaction) -> Result<()> { - // The transaction was received from P2P network so we can consider it - // propagated at this point. - let tx_hash = tx.hash; - let mut tx = tx; - tx.propagated = true; - - // Submit the tx to mempool. - self.mempool.write().await.add(tx).await?; - - //TODO copy paste of the asset manager handle_transaction method. - //added because when a tx arrive from the p2p asset are not added. - //should be done in a better way. - self.database.add_asset(&tx_hash).await - } -} - -#[async_trait::async_trait] -impl mempool::AclWhitelist for Database { - async fn contains(&self, key: &PublicKey) -> Result { - let key = entity::PublicKey(*key); - self.acl_whitelist_has(&key).await - } -} +// struct P2PTxHandler { +// mempool: Arc>, +// database: Arc, +// } + +// impl P2PTxHandler { +// pub fn new(mempool: Arc>, database: Arc) -> Self { +// Self { mempool, database } +// } +// } + +// #[async_trait::async_trait] +// impl networking::p2p::TxHandler for P2PTxHandler { +// async fn recv_tx(&self, tx: Transaction) -> Result<()> { +// // The transaction was received from P2P network so we can consider it +// // propagated at this point. +// let tx_hash = tx.hash; +// let mut tx = tx; +// tx.propagated = true; + +// // Submit the tx to mempool. +// self.mempool.write().await.add(tx).await?; + +// //TODO copy paste of the asset manager handle_transaction method. +// //added because when a tx arrive from the p2p asset are not added. +// //should be done in a better way. +// // self.database.add_asset(&tx_hash).await +// } +// } async fn run(config: Arc) -> Result<()> { let database = Arc::new(Database::new(&config.db_url).await?); let file_storage = Arc::new(storage::File::new(&config.data_directory)); + let http_peer_list: Arc>>> = + Default::default(); + + let mempool = Arc::new(RwLock::new(Mempool::new(database.clone()).await?)); + + //start Tx process event loop + let (txevent_loop_jh, tx_sender, p2p_stream) = start_event_loop( + config.data_directory.clone(), + config.p2p_listen_addr, + config.http_download_port, + http_peer_list.clone(), + database.clone(), + mempool.clone(), + ) + .await?; + let p2p = Arc::new( networking::P2P::new( "gevulot-p2p-network", @@ -206,22 +224,19 @@ async fn run(config: Arc) -> Result<()> { &config.p2p_psk_passphrase, Some(config.http_download_port), config.p2p_advertised_listen_addr, + http_peer_list, + TxEventSender::::build(tx_sender.clone()), + p2p_stream, + //database.clone() ) .await, ); - let mempool = Arc::new(RwLock::new( - Mempool::new(database.clone(), database.clone(), Some(p2p.clone())).await?, - )); - - p2p.register_tx_handler(Arc::new(P2PTxHandler::new( - mempool.clone(), - database.clone(), - ))) - .await; - - //start http download manager - let download_jh = networking::download_manager::serve_files(&config).await?; + // p2p.register_tx_handler(Arc::new(P2PTxHandler::new( + // mempool.clone(), + // database.clone(), + // ))) + // .await; // TODO(tuommaki): read total available resources from config / acquire system stats. let num_gpus = if config.gpu_devices.is_some() { 1 } else { 0 }; @@ -242,19 +257,19 @@ async fn run(config: Arc) -> Result<()> { resource_manager.clone(), ); - let asset_mgr = Arc::new(AssetManager::new( - config.clone(), - database.clone(), - p2p.as_ref().peer_http_port_list.clone(), - )); + // let asset_mgr = Arc::new(AssetManager::new( + // config.clone(), + // database.clone(), + // p2p.as_ref().peer_http_port_list.clone(), + // )); let node_key = read_node_key(&config.node_key_file)?; // Launch AssetManager's background processing. - tokio::spawn({ - let asset_mgr = asset_mgr.clone(); - async move { asset_mgr.run().await } - }); + // tokio::spawn({ + // let asset_mgr = asset_mgr.clone(); + // async move { asset_mgr.run().await } + // }); let workflow_engine = Arc::new(WorkflowEngine::new(database.clone(), file_storage.clone())); @@ -305,14 +320,14 @@ async fn run(config: Arc) -> Result<()> { let rpc_server = rpc_server::RpcServer::run( config.clone(), database.clone(), - mempool.clone(), - asset_mgr.clone(), - database.clone(), // AclWhitelist impl. + // mempool.clone(), + // asset_mgr.clone(), + TxEventSender::::build(tx_sender), ) .await?; - if let Err(err) = download_jh.await { - tracing::info!("download_manager error:{err}"); + if let Err(err) = txevent_loop_jh.await { + tracing::info!("Tx event loop error:{err}"); } Ok(()) } @@ -322,6 +337,28 @@ async fn run(config: Arc) -> Result<()> { /// others, while it doesn't participate in the Gevulot's operational side /// in any other way - i.e. this won't handle transactions in any way. async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { + let http_peer_list: Arc>>> = + Default::default(); + // //start Tx process event loop + // let (txevent_loop_jh, tx_sender, p2p_stream) = start_event_loop( + // config.data_directory.clone(), + // config.p2p_listen_addr, + // config.http_download_port, + // http_peer_list.clone(), + // Arc::new(crate::types::transaction::AlwaysGrantAclWhitelist {}), + // mempool.clone(), + // ) + // .await?; + + //build empty channel for P2P interface Transaction management. + //Indicate some domain conflict issue. + //P2P network should be started (peer domain) without Tx management (Node domain) + let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel(); + tokio::spawn(async move { while let Some(_) = rcv_tx_event_rx.recv().await {} }); + + let (_, p2p_recv) = mpsc::unbounded_channel::(); + let p2p_stream = UnboundedReceiverStream::new(p2p_recv); + let p2p = Arc::new( networking::P2P::new( "gevulot-network", @@ -329,6 +366,9 @@ async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { &config.p2p_psk_passphrase, None, config.p2p_advertised_listen_addr, + http_peer_list, + TxEventSender::::build(tx), + p2p_stream, ) .await, ); diff --git a/crates/node/src/mempool/mod.rs b/crates/node/src/mempool/mod.rs index 75b52af1..a458e927 100644 --- a/crates/node/src/mempool/mod.rs +++ b/crates/node/src/mempool/mod.rs @@ -1,6 +1,5 @@ use async_trait::async_trait; use eyre::Result; -use libsecp256k1::PublicKey; use std::collections::VecDeque; use std::sync::Arc; use thiserror::Error; @@ -18,11 +17,6 @@ pub trait Storage: Send + Sync { async fn fill_deque(&self, deque: &mut VecDeque) -> Result<()>; } -#[async_trait] -pub trait AclWhitelist: Send + Sync { - async fn contains(&self, key: &PublicKey) -> Result; -} - #[allow(clippy::enum_variant_names)] #[derive(Error, Debug)] pub enum MempoolError { @@ -33,25 +27,25 @@ pub enum MempoolError { #[derive(Clone)] pub struct Mempool { storage: Arc, - acl_whitelist: Arc, + // acl_whitelist: Arc, // TODO: This should be refactored to PubSub channel abstraction later on. - tx_chan: Option>, + // tx_chan: Option>, deque: VecDeque, } impl Mempool { pub async fn new( storage: Arc, - acl_whitelist: Arc, - tx_chan: Option>, + // acl_whitelist: Arc, + // tx_chan: Option>, ) -> Result { let mut deque = VecDeque::new(); storage.fill_deque(&mut deque).await?; Ok(Self { storage, - acl_whitelist, - tx_chan, + // acl_whitelist, + // tx_chan, deque, }) } @@ -66,29 +60,7 @@ impl Mempool { } pub async fn add(&mut self, tx: Transaction) -> Result<()> { - // First validate transaction. - tx.validate()?; - - // Secondly verify that author is whitelisted. - if !self.acl_whitelist.contains(&tx.author).await? { - return Err(MempoolError::PermissionDenied.into()); - } - - let mut tx = tx; self.storage.set(&tx).await?; - - // Broadcast new transaction to P2P network if it's configured. - if !tx.propagated { - if let Some(ref tx_chan) = self.tx_chan { - if tx_chan.send_tx(&tx).await.is_ok() { - tx.propagated = true; - self.storage.set(&tx).await?; - } else { - // TODO: Implement retry? - } - } - } - self.deque.push_back(tx); Ok(()) } diff --git a/crates/node/src/networking/mod.rs b/crates/node/src/networking/mod.rs index 0f4fea3d..aac4aeba 100644 --- a/crates/node/src/networking/mod.rs +++ b/crates/node/src/networking/mod.rs @@ -1,3 +1,2 @@ -pub mod download_manager; pub mod p2p; pub use p2p::P2P; diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index d0abf437..ed0a2c5c 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -1,3 +1,6 @@ +use crate::event_loop::P2pSender; +use crate::event_loop::TxEventSender; +use futures_util::Stream; use std::{ collections::{BTreeSet, HashMap}, io, @@ -7,6 +10,8 @@ use std::{ }; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; +use tokio::pin; +use tokio_stream::StreamExt; use super::{noise, protocol}; use bytes::{Bytes, BytesMut}; @@ -55,6 +60,9 @@ pub struct P2P { http_port: Option, nat_listen_addr: Option, psk: Vec, + + //send Tx to the process loop + tx_sender: TxEventSender, } impl Pea2Pea for P2P { @@ -70,6 +78,9 @@ impl P2P { psk_passphrase: &str, http_port: Option, nat_listen_addr: Option, + peer_http_port_list: Arc>>>, + tx_sender: TxEventSender, + propagate_tx_stream: impl Stream + std::marker::Send + 'static, ) -> Self { let config = Config { name: Some(name.into()), @@ -95,6 +106,7 @@ impl P2P { peer_http_port_list: Default::default(), http_port, nat_listen_addr, + tx_sender, }; // Enable node functionalities. @@ -103,23 +115,53 @@ impl P2P { instance.enable_writing().await; instance.enable_disconnect().await; + //start new Tx stream loop + tokio::spawn({ + let p2p = instance.clone(); + async move { + pin!(propagate_tx_stream); + while let Some(tx) = propagate_tx_stream.next().await { + let tx_hash = tx.hash; + let msg = protocol::Message::V0(protocol::MessageV0::Transaction(tx)); + let bs = match bincode::serialize(&msg) { + Ok(bs) => bs, + Err(err) => { + tracing::error!( + "Tx:{tx_hash} not send because serialization fail:{err}", + ); + continue; + } + }; + let bs = Bytes::from(bs); + tracing::debug!("broadcasting transaction {}", tx_hash); + if let Err(err) = p2p.broadcast(bs) { + tracing::error!("Tx:{tx_hash} not send because :{err}"); + } + } + } + }); + instance } - pub async fn register_tx_handler(&self, tx_handler: Arc) { - let mut old_handler = self.tx_handler.write().await; - *old_handler = tx_handler; - tracing::debug!("new tx handler registered"); - } + // pub async fn register_tx_handler(&self, tx_handler: Arc) { + // let mut old_handler = self.tx_handler.write().await; + // *old_handler = tx_handler; + // tracing::debug!("new tx handler registered"); + // } async fn recv_tx(&self, tx: Transaction) { tracing::debug!("submitting received tx to tx_handler"); - let tx_handler = self.tx_handler.read().await; - if let Err(err) = tx_handler.recv_tx(tx).await { - tracing::error!("failed to handle incoming transaction: {}", err); - } else { - tracing::debug!("submitted received tx to tx_handler"); + if let Err(err) = self.tx_sender.send_tx(tx) { + tracing::error!("P2P error during received Tx sending:{err}"); } + + // let tx_handler = self.tx_handler.read().await; + // if let Err(err) = tx_handler.recv_tx(tx).await { + // tracing::error!("failed to handle incoming transaction: {}", err); + // } else { + // tracing::debug!("submitted received tx to tx_handler"); + // } } async fn build_handshake_msg(&self) -> protocol::Handshake { diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index a31649cd..e037d46f 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -1,5 +1,8 @@ -use std::{net::SocketAddr, rc::Rc, sync::Arc}; +use crate::event_loop::RpcSender; +use crate::event_loop::TxEventSender; +use std::{net::SocketAddr, sync::Arc}; +use crate::{cli::Config, mempool::Mempool, storage::Database, types::Transaction}; use eyre::Result; use gevulot_node::types::{ rpc::{RpcError, RpcResponse}, @@ -11,19 +14,9 @@ use jsonrpsee::{ }; use tokio::sync::RwLock; -use crate::{ - asset_manager::AssetManager, - cli::Config, - mempool::{AclWhitelist, Mempool}, - storage::Database, - types::Transaction, -}; - struct Context { database: Arc, - mempool: Arc>, - asset_manager: Arc, - acl_whitelist: Arc, + tx_sender: TxEventSender, } impl std::fmt::Debug for Context { @@ -41,16 +34,12 @@ impl RpcServer { pub async fn run( cfg: Arc, database: Arc, - mempool: Arc>, - asset_manager: Arc, - acl_whitelist: Arc, + tx_sender: TxEventSender, ) -> Result { let server = Server::builder().build(cfg.json_rpc_listen_addr).await?; let mut module = RpcModule::new(Context { database, - mempool, - asset_manager, - acl_whitelist, + tx_sender, }); module.register_async_method("sendTransaction", send_transaction)?; @@ -88,43 +77,13 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp } }; - // Secondly verify that author is whitelisted. - let whitelisted = match ctx.acl_whitelist.contains(&tx.author).await { - Ok(result) => result, - Err(err) => { - tracing::error!("error when authorizing transaction: {}", err); - - // Technically following is not 100% correct, but when there is - // problem with authorizing a transaction, it cannot be passed - // through. The real nature of the problem must also not be exposed - // as it can reveal unexpected security issues. - return RpcResponse::Err(RpcError::Unauthorized); - } - }; - - if !whitelisted { - return RpcResponse::Err(RpcError::Unauthorized); - } - - // Persist transaction in DB first. - if let Err(err) = ctx.database.add_transaction(&tx).await { + if let Err(err) = ctx.tx_sender.send_tx(tx).await { + tracing::error!("failed to persist transaction: {}", err); return RpcResponse::Err(RpcError::InvalidRequest( "failed to persist transaction".to_string(), )); } - // Then add it to asset manager in order to download all necessary files - // involved. - if let Err(err) = ctx.asset_manager.handle_transaction(&tx).await { - tracing::error!( - "failed to enqueue transaction for asset processing: {}", - err - ); - return RpcResponse::Err(RpcError::InvalidRequest( - "failed to enqueue transaction for asset processing".to_string(), - )); - } - // Finally add it to mempool to propagate it to P2P network and wait // to be scheduled for execution. if let Err(err) = ctx.mempool.write().await.add(tx.clone()).await { @@ -247,14 +206,6 @@ mod tests { use super::*; - struct AlwaysGrantAclWhitelist; - #[async_trait::async_trait] - impl mempool::AclWhitelist for AlwaysGrantAclWhitelist { - async fn contains(&self, key: &PublicKey) -> Result { - Ok(true) - } - } - #[ignore] #[tokio::test] async fn test_send_transaction() { diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 7304ed65..ed846da6 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -180,49 +180,33 @@ impl Scheduler { let mut mempool = self.mempool.write().await; // Check if next tx is ready for processing? - let tx = match mempool.peek() { - Some(tx) => { - if let Payload::Run { .. } = tx.payload { - if self.database.has_assets_loaded(&tx.hash).await.unwrap() { - mempool.next().unwrap() - } else { - // Assets are still downloading. - // TODO: This can stall the whole processing pipeline!! - // XXX: ....^.........^........^......^.......^........ - tracing::info!("assets for tx {} still loading", tx.hash); - return None; - } - } else { - tracing::debug!("scheduling new task from tx {}", tx.hash); - mempool.next().unwrap() - } - } - None => return None, - }; - match self.workflow_engine.next_task(&tx).await { - Ok(res) => res, - Err(e) if e.is::() => { - let err = e.downcast_ref::(); - match err { - Some(WorkflowError::IncompatibleTransaction(_)) => { - tracing::debug!("{}", e); - None - } - _ => { - tracing::error!( - "workflow error, failed to compute next task for tx:{}: {}", - tx.hash, - e - ); - None + match mempool.next() { + Some(tx) => match self.workflow_engine.next_task(&tx).await { + Ok(res) => res, + Err(e) if e.is::() => { + let err = e.downcast_ref::(); + match err { + Some(WorkflowError::IncompatibleTransaction(_)) => { + tracing::debug!("{}", e); + None + } + _ => { + tracing::error!( + "workflow error, failed to compute next task for tx:{}: {}", + tx.hash, + e + ); + None + } } } - } - Err(e) => { - tracing::error!("failed to compute next task for tx:{}: {}", tx.hash, e); - None - } + Err(e) => { + tracing::error!("failed to compute next task for tx:{}: {}", tx.hash, e); + None + } + }, + None => None, } } diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index 9461cdb4..8710d673 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -1,16 +1,28 @@ -use std::time::Duration; - +use super::entity::{self}; +use crate::types::file::AssetFile; +use crate::types::transaction; +use crate::types::{self, transaction::ProgramData, Hash, Program, Task}; use eyre::Result; use gevulot_node::types::program::ResourceRequest; +use gevulot_node::types::transaction::TransactionError; +use libsecp256k1::PublicKey; use sqlx::{self, postgres::PgPoolOptions, FromRow, Row}; +use std::time::Duration; use uuid::Uuid; -use super::entity::{self}; -use crate::types::{self, transaction::ProgramData, File, Hash, Program, Task}; - const MAX_DB_CONNS: u32 = 64; const DB_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +#[async_trait::async_trait] +impl transaction::AclWhitelist for Database { + async fn contains(&self, key: &PublicKey) -> Result { + let key = entity::PublicKey(*key); + self.acl_whitelist_has(&key) + .await + .map_err(|err| TransactionError::General(format!("Fail to query access list: {err}",))) + } +} + #[derive(Clone)] pub struct Database { pool: sqlx::PgPool, @@ -166,10 +178,11 @@ impl Database { // Fetch accompanied Files for the Task. match task { Some(mut task) => { - let mut files = sqlx::query_as::<_, File>("SELECT * FROM file WHERE task_id = $1") - .bind(id) - .fetch_all(&mut *tx) - .await?; + let mut files = + sqlx::query_as::<_, AssetFile>("SELECT * FROM file WHERE task_id = $1") + .bind(id) + .fetch_all(&mut *tx) + .await?; task.files.append(&mut files); Ok(Some(task)) } @@ -186,7 +199,7 @@ impl Database { .await?; for task in &mut tasks { - let mut files = sqlx::query_as::<_, File>("SELECT * FROM file WHERE task_id = $1") + let mut files = sqlx::query_as::<_, AssetFile>("SELECT * FROM file WHERE task_id = $1") .bind(task.id) .fetch_all(&mut *tx) .await?; diff --git a/crates/node/src/storage/file.rs b/crates/node/src/storage/file.rs index 095cc228..fa2ba854 100644 --- a/crates/node/src/storage/file.rs +++ b/crates/node/src/storage/file.rs @@ -51,7 +51,7 @@ impl File { .join(task_id_dst) .join(path); - tracing::info!( + tracing::debug!( "moving file from {:#?} to {:#?}", src_file_path, dst_file_path diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs new file mode 100644 index 00000000..9e76f1af --- /dev/null +++ b/crates/node/src/types/file.rs @@ -0,0 +1,71 @@ +use crate::types::transaction; +use crate::types::Hash; +use serde::Deserialize; +use serde::Serialize; +use std::path::Path; +use std::path::PathBuf; + +#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] +pub struct AssetFile { + #[serde(skip_serializing, skip_deserializing)] + pub tx: Hash, + pub name: String, + pub url: String, +} + +impl AssetFile { + pub fn get_relatif_path(&self) -> PathBuf { + //TODO use file name because the checksum is not saved in the Db. + //checksum is more secure to name collision. + let file_name = Path::new(&self.name).file_name().unwrap(); + PathBuf::new().join(self.tx.to_string()).join(file_name) + } + + pub fn try_from_prg_meta_data( + value: &transaction::ProgramMetadata, + tx_hash: Hash, + ) -> Result<(Self, Hash), &'static str> { + Ok(( + AssetFile { + url: value.image_file_url.clone(), + name: value.name.clone(), + tx: tx_hash, + }, + value.image_file_checksum.clone().into(), + )) + } + + pub fn try_from_prg_data( + value: &transaction::ProgramData, + tx_hash: Hash, + ) -> Result { + let file = match value { + transaction::ProgramData::Input { + file_name, + file_url, + .. + } => AssetFile { + url: file_url.clone(), + name: file_name.clone(), + tx: tx_hash, + }, + transaction::ProgramData::Output { + source_program, + file_name, + } => { + //pick from workflow::workflow_step_to_task + todo!() + // // Make record of file that needs transfer from source tx to current tx's files. + // file_transfers.push((*source_program, file_name.clone())); + + // AssetFile { + // tx, + // name: file_name.clone(), + // url: "".to_string(), + // } + } + }; + + Ok(file) + } +} diff --git a/crates/node/src/types/mod.rs b/crates/node/src/types/mod.rs index ba477a5b..80b704b1 100644 --- a/crates/node/src/types/mod.rs +++ b/crates/node/src/types/mod.rs @@ -1,5 +1,6 @@ mod account; mod deployment; +pub mod file; mod hash; mod key_capsule; pub mod program; @@ -15,5 +16,5 @@ pub use key_capsule::KeyCapsule; pub use program::Program; pub use signature::Signature; #[allow(unused_imports)] -pub use task::{File, Task, TaskId, TaskKind, TaskResult, TaskState}; +pub use task::{Task, TaskId, TaskKind, TaskResult, TaskState}; pub use transaction::{Transaction, TransactionTree}; diff --git a/crates/node/src/types/task.rs b/crates/node/src/types/task.rs index 8503e7f5..9c68b2d8 100644 --- a/crates/node/src/types/task.rs +++ b/crates/node/src/types/task.rs @@ -1,6 +1,5 @@ +use crate::types::file::AssetFile; use serde::{Deserialize, Serialize}; -use std::path::Path; -use std::path::PathBuf; use uuid::Uuid; use super::hash::{deserialize_hash_from_json, Hash}; @@ -28,21 +27,6 @@ pub enum TaskKind { Nop, } -#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] -pub struct File { - #[serde(skip_serializing, skip_deserializing)] - pub tx: Hash, - pub name: String, - pub url: String, -} - -impl File { - pub fn get_file_relative_path(&self) -> PathBuf { - let file_name = Path::new(&self.name).file_name().unwrap(); - PathBuf::new().join(self.tx.to_string()).join(file_name) - } -} - #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] pub struct Task { pub id: TaskId, @@ -53,7 +37,7 @@ pub struct Task { pub program_id: Hash, pub args: Vec, #[sqlx(skip)] - pub files: Vec, + pub files: Vec, #[serde(skip_deserializing)] pub serial: i32, #[serde(skip_deserializing)] diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 1f1c43b4..eee2d305 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -1,14 +1,29 @@ -use std::{collections::HashSet, rc::Rc}; - +use super::file::AssetFile; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; +use crate::types::transaction; +use async_trait::async_trait; use eyre::Result; use libsecp256k1::{sign, verify, Message, PublicKey, SecretKey}; use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use sha3::{Digest, Sha3_256}; +use std::{collections::HashSet, rc::Rc}; use thiserror::Error; +#[async_trait] +pub trait AclWhitelist: Send + Sync { + async fn contains(&self, key: &PublicKey) -> Result; +} + +pub struct AlwaysGrantAclWhitelist; +#[async_trait::async_trait] +impl AclWhitelist for AlwaysGrantAclWhitelist { + async fn contains(&self, _key: &PublicKey) -> Result { + Ok(true) + } +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub enum TransactionTree { Root { @@ -172,7 +187,71 @@ pub enum Payload { } impl Payload { - fn serialize_into(&self, buf: &mut Vec) { + pub fn get_asset_list(&self, tx_hash: Hash) -> Result> { + match self { + transaction::Payload::Deploy { + prover, verifier, .. + } => Ok(vec![ + AssetFile::try_from_prg_meta_data(prover, tx_hash).map_err(|err| { + TransactionError::Validation(format!("Fail to get prover image file: {err}",)) + })?, + AssetFile::try_from_prg_meta_data(verifier, tx_hash).map_err(|err| { + TransactionError::Validation( + format!("Fail to gfet verifier image file: {err}",), + ) + })?, + ]), + Payload::Run { workflow } => { + workflow + .steps + .iter() + .flat_map(|step| &step.inputs) + .filter_map(|input| { + match input { + ProgramData::Input { + file_name, + file_url, + checksum, + } => Some((file_name, file_url, checksum)), + ProgramData::Output { .. } => { + /* ProgramData::Output as input means it comes from another + program execution -> skip this branch. */ + None + } + } + }) + .map(|(file_name, file_url, checksum)| { + //verify the url is valide. + reqwest::Url::parse(&file_url)?; + Ok(( + AssetFile { + url: file_url.clone(), + name: file_name.to_string(), + tx: tx_hash, + // checksum: checksum.to_string().into(), + }, + checksum.to_string().into(), + )) + }) + .collect() + } + Payload::Proof { + parent, + prover, + proof, + } => todo!(), + Payload::Verification { + parent, + verifier, + verification, + } => todo!(), + // Other transaction types don't have external assets that would + // need processing. + _ => Ok(vec![]), + } + } + + pub fn serialize_into(&self, buf: &mut Vec) { match self { Payload::Empty => {} Payload::Transfer { to, value } => { @@ -231,6 +310,8 @@ impl Payload { pub enum TransactionError { #[error("validation: {0}")] Validation(String), + #[error("General error: {0}")] + General(String), } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] @@ -303,7 +384,7 @@ impl Transaction { (&hasher.finalize()[0..32]).into() } - pub fn validate(&self) -> Result<()> { + pub fn validate(&self) -> Result<(), TransactionError> { if let Payload::Run { ref workflow } = self.payload { let mut programs = HashSet::new(); for step in &workflow.steps { diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index 967649d2..fce95494 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -1,10 +1,11 @@ +use crate::types::file::AssetFile; use std::sync::Arc; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::{ transaction::{Payload, ProgramData, Workflow, WorkflowStep}, - File, Hash, Task, TaskKind, Transaction, + Hash, Task, TaskKind, Transaction, }; use thiserror::Error; use uuid::Uuid; @@ -25,6 +26,9 @@ pub enum WorkflowError { #[error("transaction not found: {0}")] TransactionNotFound(Hash), + + #[error("Program file definition error: {0}")] + FileDefinitionError(String), } #[async_trait] @@ -204,9 +208,9 @@ impl WorkflowEngine { cur_tx = parent; continue; } - _ => { + payload => { tracing::debug!( - "failed to find workflow for transaction {}: incompatible transaction", + "Find parent failed to find workflow for transaction {}: incompatible transaction: {payload:?}", cur_tx ); return Err(WorkflowError::IncompatibleTransaction(cur_tx.to_string()).into()); @@ -249,9 +253,9 @@ impl WorkflowEngine { tx_hash = parent; continue; } - _ => { + payload => { tracing::debug!( - "failed to find workflow for transaction {}: incompatible transaction", + "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", &tx_hash ); return Err(WorkflowError::IncompatibleTransaction(tx_hash.to_string()).into()); @@ -267,37 +271,41 @@ impl WorkflowEngine { kind: TaskKind, ) -> Result { let id = Uuid::new_v4(); - let mut file_transfers = vec![]; + let mut file_transfers: Vec<(Hash, String)> = vec![]; let files = step .inputs .iter() - .map(|e| match e { - ProgramData::Input { - file_name, - file_url, - .. - } => File { - tx, - name: file_name.clone(), - url: file_url.clone(), - }, - ProgramData::Output { - source_program, - file_name, - } => { - // Make record of file that needs transfer from source tx to current tx's files. - file_transfers.push((*source_program, file_name.clone())); - - File { - tx, - name: file_name.clone(), - url: "".to_string(), - } - } + .map(|e| { + AssetFile::try_from_prg_data(e, tx) + .map_err(|err| WorkflowError::FileDefinitionError(err.to_string()).into()) }) - .collect(); + // match e { + // ProgramData::Input { + // file_name, + // file_url, + // .. + // } => File { + // tx, + // name: file_name.clone(), + // url: file_url.clone(), + // }, + // ProgramData::Output { + // source_program, + // file_name, + // } => { + // // Make record of file that needs transfer from source tx to current tx's files. + // file_transfers.push((*source_program, file_name.clone())); + // File { + // tx, + // name: file_name.clone(), + // url: "".to_string(), + // } + // } + // }) + .collect::>>()?; // Process file transfers from source programs. + //TODO! move end task execution. for (source_program, file_name) in file_transfers { let source_tx = self .find_parent_tx_for_program(&tx, &source_program) From 00831fd54d2eb2ea35129a83abf7bbf8f8105643 Mon Sep 17 00:00:00 2001 From: musitdev Date: Thu, 8 Feb 2024 18:54:54 +0100 Subject: [PATCH 02/43] correct file management. Work locally. Before remote test --- Cargo.lock | 1 + .../migrations/20231009111925_tasks-table.sql | 1 + .../20231128120351_transactions-table.sql | 11 + crates/node/src/asset_manager/mod.rs | 13 +- crates/node/src/event_loop/mod.rs | 402 ------------------ crates/node/src/main.rs | 25 +- crates/node/src/networking/p2p/mod.rs | 2 +- crates/node/src/networking/p2p/pea2pea.rs | 4 +- crates/node/src/rpc_server/mod.rs | 7 +- crates/node/src/scheduler/mod.rs | 91 ++++ crates/node/src/storage/database/postgres.rs | 66 ++- .../download_manager.rs | 4 +- crates/node/src/txvalidation/event.rs | 229 ++++++++++ crates/node/src/txvalidation/mod.rs | 198 +++++++++ crates/node/src/types/file.rs | 156 +++++-- crates/node/src/types/task.rs | 4 +- crates/node/src/types/transaction.rs | 66 +-- crates/node/src/vmm/qemu.rs | 2 +- crates/node/src/vmm/vm_server.rs | 18 + crates/node/src/workflow/mod.rs | 48 +-- crates/shim/Cargo.toml | 1 + crates/shim/src/lib.rs | 25 +- 22 files changed, 839 insertions(+), 535 deletions(-) delete mode 100644 crates/node/src/event_loop/mod.rs rename crates/node/src/{event_loop => txvalidation}/download_manager.rs (98%) create mode 100644 crates/node/src/txvalidation/event.rs create mode 100644 crates/node/src/txvalidation/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 51af7db4..4ee856de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1203,6 +1203,7 @@ dependencies = [ "anyhow", "async-stream", "prost 0.11.9", + "sha3", "tokio", "tokio-stream", "tokio-vsock", diff --git a/crates/node/migrations/20231009111925_tasks-table.sql b/crates/node/migrations/20231009111925_tasks-table.sql index 1c2ce845..0ecbd9be 100644 --- a/crates/node/migrations/20231009111925_tasks-table.sql +++ b/crates/node/migrations/20231009111925_tasks-table.sql @@ -24,6 +24,7 @@ CREATE TABLE file ( task_id uuid NOT NULL, name VARCHAR(256) NOT NULL, url VARCHAR(2048) NOT NULL, + checksum VARCHAR(64) NOT NULL, CONSTRAINT fk_task FOREIGN KEY (task_id) REFERENCES task (id) ON DELETE CASCADE diff --git a/crates/node/migrations/20231128120351_transactions-table.sql b/crates/node/migrations/20231128120351_transactions-table.sql index 8ce61ec2..34dfbd9c 100644 --- a/crates/node/migrations/20231128120351_transactions-table.sql +++ b/crates/node/migrations/20231128120351_transactions-table.sql @@ -117,3 +117,14 @@ CREATE TABLE proof_key ( FOREIGN KEY (tx) REFERENCES transaction (hash) ON DELETE CASCADE ); + + +CREATE TABLE txfile ( + tx_id VARCHAR(64) NOT NULL, + name VARCHAR(256) NOT NULL, + url VARCHAR(2048) NOT NULL, + checksum VARCHAR(64) NOT NULL, + CONSTRAINT fk_tx + FOREIGN KEY (tx_id) + REFERENCES transaction (hash) ON DELETE CASCADE +); diff --git a/crates/node/src/asset_manager/mod.rs b/crates/node/src/asset_manager/mod.rs index 3653b57a..66bc49c3 100644 --- a/crates/node/src/asset_manager/mod.rs +++ b/crates/node/src/asset_manager/mod.rs @@ -2,19 +2,12 @@ use crate::{ cli::Config, storage::Database, - types::{ - transaction::{self, Transaction}, - Hash, Program, - }, -}; -use eyre::{eyre, Result}; -use gevulot_node::types::{ - self, - transaction::{Payload, ProgramData}, + types::{transaction::Transaction, Hash}, }; +use eyre::Result; use std::collections::HashMap; use std::net::SocketAddr; -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; use thiserror::Error; use tokio::time::sleep; diff --git a/crates/node/src/event_loop/mod.rs b/crates/node/src/event_loop/mod.rs deleted file mode 100644 index 9e3dd869..00000000 --- a/crates/node/src/event_loop/mod.rs +++ /dev/null @@ -1,402 +0,0 @@ -use crate::types::transaction::Payload; -use crate::types::transaction::TransactionError; -use crate::types::Hash; -use crate::types::Signature; -use crate::types::Transaction; -use crate::Mempool; -use futures::future::join_all; -use futures_util::stream::FuturesUnordered; -use futures_util::Stream; -use futures_util::StreamExt; -use futures_util::TryFutureExt; -use gevulot_node::types::transaction::AclWhitelist; -use libsecp256k1::verify; -use libsecp256k1::Message; -use libsecp256k1::PublicKey; -use sha3::{Digest, Sha3_256}; -use std::collections::HashMap; -use std::collections::HashSet; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::net::SocketAddr; -use std::path::PathBuf; -use std::sync::Arc; -use thiserror::Error; -use tokio::sync::mpsc; -use tokio::sync::mpsc::UnboundedReceiver; -use tokio::sync::mpsc::UnboundedSender; -use tokio::sync::oneshot; -use tokio::sync::RwLock; -use tokio::task::JoinHandle; -use tokio_stream::wrappers::UnboundedReceiverStream; - -mod download_manager; - -#[derive(Error, Debug)] -pub enum EventProcessError { - #[error("Fail to rcv Tx from the channel: {0}")] - RcvChannelError(#[from] tokio::sync::oneshot::error::RecvError), - #[error("Fail to send the Tx on the channel: {0}")] - SendChannelError(#[from] tokio::sync::mpsc::error::SendError<(RcvTx, Option)>), - #[error("Fail to send the Tx on the channel: {0}")] - PropagateTxError(#[from] tokio::sync::mpsc::error::SendError), - #[error("validation fail: {0}")] - ValidateError(String), - #[error("Tx asset fail to download because {0}")] - DownloadAssetError(String), - #[error("Save Tx error: {0}")] - SaveTxError(String), -} - -pub type CallbackSender = oneshot::Sender>; - -///Content of all Tx. -#[derive(Debug, Clone)] -struct EventTxContent { - pub author: PublicKey, - pub hash: Hash, - pub payload: Payload, - pub nonce: u64, - pub signature: Signature, -} - -impl From for EventTxContent { - fn from(tx: Transaction) -> Self { - EventTxContent { - author: tx.author, - hash: tx.hash, - payload: tx.payload, - nonce: tx.nonce, - signature: tx.signature, - } - } -} - -impl From for Transaction { - fn from(content: EventTxContent) -> Self { - Transaction { - author: content.author, - hash: content.hash, - payload: content.payload, - nonce: content.nonce, - signature: content.signature, - propagated: true, - executed: false, - } - } -} - -//event base type. -#[derive(Debug, Clone)] -pub struct EventTx { - content: EventTxContent, - tx_type: T, -} - -impl EventTx { - fn transition_new(&self, tx_type: N) -> EventTx { - EventTx { - content: self.content.clone(), - tx_type, - } - } - fn transition(self, tx_type: N) -> EventTx { - EventTx { - content: self.content, - tx_type, - } - } -} - -#[derive(Debug)] -enum SourceTxType { - P2P, - RPC, - TXRESULT, -} - -impl EventTx { - async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { - let tx: Transaction = self.content.clone().into(); - tx.validate()?; - - // Secondly verify that author is whitelisted. - if !acl_whitelist.contains(&tx.author).await? { - return Err(TransactionError::Validation( - "Tx permission denied signer not authorized".to_string(), - )); - } - - if let Payload::Run { ref workflow } = self.content.payload { - let mut programs = HashSet::new(); - for step in &workflow.steps { - if !programs.insert(step.program) { - return Err(TransactionError::Validation(format!( - "multiple programs in workflow: {}", - &step.program - ))); - } - } - } - - Ok(()) - } - - fn verify_tx_signature(&self) -> bool { - let mut hasher = Sha3_256::new(); - let mut buf = vec![]; - hasher.update(self.content.author.serialize()); - self.content.payload.serialize_into(&mut buf); - hasher.update(buf); - hasher.update(self.content.nonce.to_be_bytes()); - - let hash: Hash = (&hasher.finalize()[0..32]).into(); - let msg: Message = hash.into(); - verify(&msg, &self.content.signature.into(), &self.content.author) - } - - fn propagate(self) -> Option { - match self.tx_type { - SourceTxType::P2P => None, - SourceTxType::RPC => Some(PropagateTx(EventTx { - content: self.content, - tx_type: ConsistentTx, - })), - SourceTxType::TXRESULT => Some(PropagateTx(EventTx { - content: self.content, - tx_type: ConsistentTx, - })), - } - } -} - -#[derive(Debug)] -struct ConsistentTx; - -//event list -pub struct RcvTx(EventTx); -impl RcvTx { - async fn process_event( - self, - acl_whitelist: &impl AclWhitelist, - ) -> Result { - match self.0.validate_tx(acl_whitelist).await { - Ok(()) => Ok(DownloadTx(self.0)), - Err(err) => Err(EventProcessError::ValidateError(format!( - "Tx validation fail:{err}" - ))), - } - } -} -struct DownloadTx(EventTx); -impl DownloadTx { - async fn process_event( - self, - local_directory_path: &PathBuf, - http_peer_list: Vec<(SocketAddr, Option)>, - ) -> Result<(NewTx, Option), EventProcessError> { - let tx_hash = self.0.content.hash; - let http_client = reqwest::Client::new(); - let asset_file_list = self - .0 - .content - .payload - .get_asset_list(tx_hash) - .map_err(|err| { - EventProcessError::DownloadAssetError(format!( - "Asset file param conversion error:{err}" - )) - })?; - - let futures: Vec<_> = asset_file_list - .into_iter() - .map(|(asset_file, checksum)| { - download_manager::download_asset_file( - local_directory_path, - &http_peer_list, - &http_client, - asset_file, - checksum, - ) - }) - .collect(); - join_all(futures) - .await - .into_iter() - .map(|res| res) - .collect::, _>>() - .map_err(|err| { - EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) - })?; - let newtx = NewTx(self.0.transition_new(ConsistentTx)); - let propagate = self.0.propagate(); - Ok((newtx, propagate)) - } -} -struct PropagateTx(EventTx); -impl PropagateTx { - async fn process_event( - self, - p2p_sender: &UnboundedSender, - ) -> Result<(), EventProcessError> { - let tx: Transaction = self.0.content.into(); - p2p_sender.send(tx).map_err(|err| err.into()) - } -} - -struct NewTx(EventTx); -impl NewTx { - async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { - let tx: Transaction = self.0.content.into(); - mempool - .add(tx) - .map_err(|err| EventProcessError::SaveTxError(format!("{err}"))) - .await - } -} - -//send Tx interface with outside the module -pub struct RpcSender; -#[derive(Clone)] -pub struct P2pSender; -pub struct TxResultSender; - -#[derive(Debug, Clone)] -pub struct TxEventSender { - sender: UnboundedSender<(RcvTx, Option)>, - _marker: PhantomData, -} - -impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { - TxEventSender { - sender, - _marker: PhantomData, - } - } - - pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::P2P, - }; - self.sender - .send((RcvTx(event), None)) - .map_err(|err| err.into()) - } -} - -impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { - TxEventSender { - sender, - _marker: PhantomData, - } - } - - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { - let (sender, rx) = oneshot::channel(); - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::RPC, - }; - self.sender - .send((RcvTx(event), Some(sender))) - .map_err(|err| EventProcessError::from(err))?; - rx.await? - } -} - -impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { - TxEventSender { - sender, - _marker: PhantomData, - } - } - - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { - let (sender, rx) = oneshot::channel(); - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::TXRESULT, - }; - self.sender - .send((RcvTx(event), Some(sender))) - .map_err(|err| EventProcessError::from(err))?; - rx.await? - } -} - -pub async fn start_event_loop( - local_directory_path: PathBuf, - bind_addr: SocketAddr, - http_download_port: u16, - http_peer_list: Arc>>>, - acl_whitelist: Arc, - mempool: Arc>, -) -> eyre::Result<( - JoinHandle<()>, - UnboundedSender<(RcvTx, Option)>, - impl Stream, -)> { - let local_directory_path = Arc::new(local_directory_path); - //start http download manager - let download_jh = - download_manager::serve_files(bind_addr, http_download_port, local_directory_path.clone()) - .await?; - - let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel::<(RcvTx, Option)>(); - - let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::(); - let p2p_stream = UnboundedReceiverStream::new(p2p_recv); - let jh = tokio::spawn({ - let local_directory_path = local_directory_path.clone(); - - async move { - while let Some((event, callback)) = rcv_tx_event_rx.recv().await { - //process RcvTx(EventTx) event - let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; - tokio::spawn({ - let p2p_sender = p2p_sender.clone(); - let local_directory_path = local_directory_path.clone(); - let acl_whitelist = acl_whitelist.clone(); - let mempool = mempool.clone(); - async move { - let res = event - .process_event(acl_whitelist.as_ref()) - .and_then(|download_event| { - download_event.process_event(&local_directory_path, http_peer_list) - }) - .and_then(|(new_tx, propagate_tx)| async move { - if let Some(propagate_tx) = propagate_tx { - propagate_tx.process_event(&p2p_sender).await?; - } - new_tx.process_event(&mut *(mempool.write().await)).await?; - - Ok(()) - }) - .await; - //send the execution result back if needed. - if let Some(callback) = callback { - //forget the result because if the RPC connection is closed the send can fail. - let _ = callback.send(res); - } - } - }); - } - } - }); - Ok((jh, tx, p2p_stream)) -} - -async fn convert_peer_list_to_vec( - http_peer_list: &tokio::sync::RwLock>>, -) -> Vec<(SocketAddr, Option)> { - http_peer_list - .read() - .await - .iter() - .map(|(a, p)| (*a, *p)) - .collect() -} diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 8c87426e..21f38570 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -1,10 +1,6 @@ #![allow(dead_code)] #![allow(unused_variables)] -use crate::event_loop::start_event_loop; -use crate::event_loop::P2pSender; -use crate::event_loop::RpcSender; -use crate::event_loop::TxEventSender; use async_trait::async_trait; use clap::Parser; use cli::{ @@ -37,13 +33,13 @@ use workflow::WorkflowEngine; mod asset_manager; mod cli; -mod event_loop; mod mempool; mod nanos; mod networking; mod rpc_server; mod scheduler; mod storage; +mod txvalidation; mod vmm; mod workflow; @@ -199,7 +195,6 @@ impl workflow::TransactionStore for storage::Database { async fn run(config: Arc) -> Result<()> { let database = Arc::new(Database::new(&config.db_url).await?); - let file_storage = Arc::new(storage::File::new(&config.data_directory)); let http_peer_list: Arc>>> = Default::default(); @@ -207,7 +202,7 @@ async fn run(config: Arc) -> Result<()> { let mempool = Arc::new(RwLock::new(Mempool::new(database.clone()).await?)); //start Tx process event loop - let (txevent_loop_jh, tx_sender, p2p_stream) = start_event_loop( + let (txevent_loop_jh, tx_sender, p2p_stream) = txvalidation::start_event_loop( config.data_directory.clone(), config.p2p_listen_addr, config.http_download_port, @@ -225,7 +220,7 @@ async fn run(config: Arc) -> Result<()> { Some(config.http_download_port), config.p2p_advertised_listen_addr, http_peer_list, - TxEventSender::::build(tx_sender.clone()), + txvalidation::TxEventSender::::build(tx_sender.clone()), p2p_stream, //database.clone() ) @@ -271,7 +266,12 @@ async fn run(config: Arc) -> Result<()> { // async move { asset_mgr.run().await } // }); - let workflow_engine = Arc::new(WorkflowEngine::new(database.clone(), file_storage.clone())); + let workflow_engine = Arc::new(WorkflowEngine::new(database.clone())); + let download_url_prefix = format!( + "http://{}:{}", + config.p2p_listen_addr.to_string(), + config.http_download_port + ); let scheduler = Arc::new(scheduler::Scheduler::new( mempool.clone(), @@ -279,8 +279,11 @@ async fn run(config: Arc) -> Result<()> { program_manager, workflow_engine, node_key, + config.data_directory.clone(), + download_url_prefix, )); + let file_storage = Arc::new(storage::File::new(&config.data_directory)); let vm_server = vmm::vm_server::VMServer::new(scheduler.clone(), provider, file_storage.clone()); @@ -322,7 +325,7 @@ async fn run(config: Arc) -> Result<()> { database.clone(), // mempool.clone(), // asset_mgr.clone(), - TxEventSender::::build(tx_sender), + txvalidation::TxEventSender::::build(tx_sender), ) .await?; @@ -367,7 +370,7 @@ async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { None, config.p2p_advertised_listen_addr, http_peer_list, - TxEventSender::::build(tx), + txvalidation::TxEventSender::::build(tx), p2p_stream, ) .await, diff --git a/crates/node/src/networking/p2p/mod.rs b/crates/node/src/networking/p2p/mod.rs index ed8c2dcf..7afa5bc3 100644 --- a/crates/node/src/networking/p2p/mod.rs +++ b/crates/node/src/networking/p2p/mod.rs @@ -1,4 +1,4 @@ mod noise; mod pea2pea; mod protocol; -pub use pea2pea::{TxChannel, TxHandler, P2P}; +pub use pea2pea::{TxHandler, P2P}; diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index ed0a2c5c..5c82f5dd 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -1,5 +1,5 @@ -use crate::event_loop::P2pSender; -use crate::event_loop::TxEventSender; +use crate::txvalidation::P2pSender; +use crate::txvalidation::TxEventSender; use futures_util::Stream; use std::{ collections::{BTreeSet, HashMap}, diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index e037d46f..fd547c6d 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -1,8 +1,8 @@ -use crate::event_loop::RpcSender; -use crate::event_loop::TxEventSender; +use crate::txvalidation::RpcSender; +use crate::txvalidation::TxEventSender; use std::{net::SocketAddr, sync::Arc}; -use crate::{cli::Config, mempool::Mempool, storage::Database, types::Transaction}; +use crate::{cli::Config, storage::Database, types::Transaction}; use eyre::Result; use gevulot_node::types::{ rpc::{RpcError, RpcResponse}, @@ -12,7 +12,6 @@ use jsonrpsee::{ server::{RpcModule, Server, ServerHandle}, types::Params, }; -use tokio::sync::RwLock; struct Context { database: Arc, diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index ed846da6..eb167dae 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -2,6 +2,7 @@ mod program_manager; mod resource_manager; use crate::storage::Database; +use crate::types::file::{move_vmfile, TxFile, VmFile}; use crate::types::TaskState; use crate::vmm::vm_server::grpc; use crate::vmm::{vm_server::TaskManager, VMId}; @@ -13,6 +14,8 @@ use libsecp256k1::SecretKey; pub use program_manager::ProgramManager; use rand::RngCore; pub use resource_manager::ResourceManager; +use std::path::PathBuf; +use uuid::Uuid; use std::time::Instant; use std::{ @@ -63,6 +66,8 @@ pub struct Scheduler { running_vms: Arc>>, #[allow(clippy::type_complexity)] task_queue: Arc>>>, + data_directory: PathBuf, + http_download_host: String, } impl Scheduler { @@ -72,6 +77,8 @@ impl Scheduler { program_manager: ProgramManager, workflow_engine: Arc, node_key: SecretKey, + data_directory: PathBuf, + http_download_host: String, ) -> Self { Self { mempool, @@ -84,6 +91,8 @@ impl Scheduler { running_tasks: Arc::new(Mutex::new(vec![])), running_vms: Arc::new(Mutex::new(vec![])), task_queue: Arc::new(Mutex::new(HashMap::new())), + data_directory, + http_download_host, } } @@ -407,6 +416,76 @@ impl TaskManager for Scheduler { ); } + //move and build output files of the Tx execution + let executed_files: Vec<(VmFile, TxFile)> = result + .files + .iter() + .map(|file| { + let vm_file = VmFile { + task_id: running_task.task.tx, + name: file.path.to_string(), + }; + + let uuid = Uuid::new_v4(); + //format file_name to keep the current filename and the uuid. + //put uuid at the end so that the uuid is use to save the file. + let new_file_name = format!("{}/{}", file.path, uuid); + let dest = TxFile { + //Real url will be calculated during download. + url: self.http_download_host.clone(), + name: new_file_name, + checksum: file.checksum[..].into(), + }; + (vm_file, dest) + }) + .collect(); + + //Verify that all expected files has been generated. + // let executed_files: Vec<(VmFile, TxFile)> = + // if let Ok(Some(tx)) = self.database.find_transaction(&running_task.task.tx).await { + // if let Payload::Run { workflow } = tx.payload { + // workflow + // .steps + // .iter() + // .flat_map(|step| &step.inputs) + // .filter_map(|input| { + // let ProgramData::Output { + // source_program, + // file_name, + // } = input + // else { + // return None; + // }; + // let vm_file = VmFile { + // task_id: running_task.task.tx, + // name: file_name.to_string(), + // }; + + // let uuid = Uuid::new_v4(); + // //format file_name to keep the current filename and the uuid. + // //put uuid at the end so that the uuid is use to save the file. + // let new_file_name = format!("{}/{}", file_name, uuid); + // let dest = TxFile { + // //Real url will be calculated during download. + // url: self.http_download_host.clone(), + // name: new_file_name, + // checksum: Hash::default(), + // }; + // Some((vm_file, dest)) + // }) + // .collect() + // } else { + // vec![] + // } + // } else { + // vec![] + // }; + + let new_tx_files: Vec = executed_files + .iter() + .map(|(_, file)| file) + .cloned() + .collect(); let nonce = rand::thread_rng().next_u64(); let tx = match running_task.task.kind { TaskKind::Proof => Transaction::new( @@ -414,6 +493,7 @@ impl TaskManager for Scheduler { parent: running_task.task.tx, prover: program, proof: result.data, + files: new_tx_files, }, &self.node_key, ), @@ -422,6 +502,7 @@ impl TaskManager for Scheduler { parent: running_task.task.tx, verifier: program, verification: result.data, + files: new_tx_files, }, &self.node_key, ), @@ -436,6 +517,16 @@ impl TaskManager for Scheduler { } }; + //Move tx file from execution Tx path to new Tx path + for (source_file, dest_file) in executed_files { + let dest = dest_file.to_asset_file(tx.hash); + if let Err(err) = move_vmfile(&source_file, &dest, &self.data_directory).await { + tracing::error!( + "failed to move excution file from: {source_file:?} to: {dest:?} error: {err}", + ); + } + } + let mut mempool = self.mempool.write().await; if let Err(err) = mempool.add(tx.clone()).await { tracing::error!("failed to add transaction to mempool: {}", err); diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index 8710d673..c2a87a4e 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -1,5 +1,5 @@ use super::entity::{self}; -use crate::types::file::AssetFile; +use crate::types::file::TxFile; use crate::types::transaction; use crate::types::{self, transaction::ProgramData, Hash, Program, Task}; use eyre::Result; @@ -149,11 +149,12 @@ impl Database { { let mut query_builder = - sqlx::QueryBuilder::new("INSERT INTO file ( task_id, name, url )"); + sqlx::QueryBuilder::new("INSERT INTO file ( task_id, name, url, checksum )"); query_builder.push_values(&t.files, |mut b, new_file| { b.push_bind(t.id) .push_bind(&new_file.name) - .push_bind(&new_file.url); + .push_bind(&new_file.url) + .push_bind(&new_file.checksum); }); let query = query_builder.build(); @@ -179,7 +180,7 @@ impl Database { match task { Some(mut task) => { let mut files = - sqlx::query_as::<_, AssetFile>("SELECT * FROM file WHERE task_id = $1") + sqlx::query_as::<_, TxFile>("SELECT * FROM file WHERE task_id = $1") .bind(id) .fetch_all(&mut *tx) .await?; @@ -199,7 +200,7 @@ impl Database { .await?; for task in &mut tasks { - let mut files = sqlx::query_as::<_, AssetFile>("SELECT * FROM file WHERE task_id = $1") + let mut files = sqlx::query_as::<_, TxFile>("SELECT * FROM file WHERE task_id = $1") .bind(task.id) .fetch_all(&mut *tx) .await?; @@ -361,6 +362,13 @@ impl Database { } } entity::transaction::Kind::Proof => { + //get payload files + let files = + sqlx::query_as::<_, TxFile>("SELECT * FROM txfile WHERE tx_id = $1") + .bind(tx_hash) + .fetch_all(&mut *db_tx) + .await?; + sqlx::query("SELECT parent, prover, proof FROM proof WHERE tx = $1") .bind(tx_hash) .map( @@ -368,6 +376,7 @@ impl Database { parent: row.get(0), prover: row.get(1), proof: row.get(2), + files: files.clone(), //to avoid file move. }, ) .fetch_one(&mut *db_tx) @@ -386,6 +395,12 @@ impl Database { .await? } entity::transaction::Kind::Verification => { + //get payload files + let files = + sqlx::query_as::<_, TxFile>("SELECT * FROM txfile WHERE tx_id = $1") + .bind(tx_hash) + .fetch_all(&mut *db_tx) + .await?; sqlx::query( "SELECT parent, verifier, verification FROM verification WHERE tx = $1", ) @@ -395,6 +410,7 @@ impl Database { parent: row.get(0), verifier: row.get(1), verification: row.get(2), + files: files.clone(), //to avoid file move. }, ) .fetch_one(&mut *db_tx) @@ -552,6 +568,7 @@ impl Database { parent, prover, proof, + files, } => { sqlx::query( "INSERT INTO proof ( tx, parent, prover, proof ) VALUES ( $1, $2, $3, $4 ) ON CONFLICT (tx) DO NOTHING", @@ -562,6 +579,25 @@ impl Database { .bind(proof) .execute(&mut *db_tx) .await?; + + //save payload files + if !files.is_empty() { + let mut query_builder = sqlx::QueryBuilder::new( + "INSERT INTO txfile ( tx_id, name, url, checksum )", + ); + query_builder.push_values(files, |mut b, new_file| { + b.push_bind(tx.hash) + .push_bind(&new_file.name) + .push_bind(&new_file.url) + .push_bind(&new_file.checksum); + }); + + let query = query_builder.build(); + if let Err(err) = query.execute(&mut *db_tx).await { + db_tx.rollback().await?; + return Err(err.into()); + } + } } types::transaction::Payload::ProofKey { parent, key } => { @@ -577,6 +613,7 @@ impl Database { parent, verifier, verification, + files, } => { sqlx::query( "INSERT INTO verification ( tx, parent, verifier, verification ) VALUES ( $1, $2, $3, $4 ) ON CONFLICT (tx) DO NOTHING", @@ -587,6 +624,25 @@ impl Database { .bind(verification) .execute(&mut *db_tx) .await?; + + //save payload files + if !files.is_empty() { + let mut query_builder = sqlx::QueryBuilder::new( + "INSERT INTO txfile ( tx_id, name, url, checksum )", + ); + query_builder.push_values(files, |mut b, new_file| { + b.push_bind(tx.hash) + .push_bind(&new_file.name) + .push_bind(&new_file.url) + .push_bind(&new_file.checksum); + }); + + let query = query_builder.build(); + if let Err(err) = query.execute(&mut *db_tx).await { + db_tx.rollback().await?; + return Err(err.into()); + } + } } _ => { /* ignore for now */ } } diff --git a/crates/node/src/event_loop/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs similarity index 98% rename from crates/node/src/event_loop/download_manager.rs rename to crates/node/src/txvalidation/download_manager.rs index 9ad2dde0..06f7e358 100644 --- a/crates/node/src/event_loop/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -27,8 +27,6 @@ pub async fn download_asset_file( http_peer_list: &[(SocketAddr, Option)], http_client: &reqwest::Client, asset_file: AssetFile, - asset_checksum: gevulot_node::types::Hash, - // file_hash: gevulot_node::types::Hash, ) -> Result<()> { let local_relative_file_path = asset_file.get_relatif_path(); tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); @@ -90,7 +88,7 @@ pub async fn download_asset_file( fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); - if checksum != asset_checksum { + if checksum != asset_file.checksum { Err(eyre!("Download file: {:?}, bad checksum", asset_file.name)) } else { //rename to original name diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs new file mode 100644 index 00000000..e127c144 --- /dev/null +++ b/crates/node/src/txvalidation/event.rs @@ -0,0 +1,229 @@ +use crate::txvalidation::download_manager; +use crate::txvalidation::EventProcessError; +use crate::types::transaction::Payload; +use crate::types::transaction::TransactionError; +use crate::types::Hash; +use crate::types::Signature; +use crate::types::Transaction; +use crate::Mempool; +use futures::future::join_all; +use futures_util::TryFutureExt; +use gevulot_node::types::transaction::AclWhitelist; +use libsecp256k1::verify; +use libsecp256k1::Message; +use libsecp256k1::PublicKey; +use sha3::{Digest, Sha3_256}; +use std::collections::HashSet; +use std::fmt::Debug; +use std::net::SocketAddr; +use std::path::PathBuf; +use tokio::sync::mpsc::UnboundedSender; + +///Content of all Tx. +#[derive(Debug, Clone)] +pub struct EventTxContent { + pub author: PublicKey, + pub hash: Hash, + pub payload: Payload, + pub nonce: u64, + pub signature: Signature, +} + +impl From for EventTxContent { + fn from(tx: Transaction) -> Self { + EventTxContent { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + } + } +} + +impl From for Transaction { + fn from(content: EventTxContent) -> Self { + Transaction { + author: content.author, + hash: content.hash, + payload: content.payload, + nonce: content.nonce, + signature: content.signature, + propagated: true, + executed: false, + } + } +} + +//event base type. + +//Marker type to define the event. +#[derive(Debug)] +pub enum SourceTxType { + P2P, + RPC, + TXRESULT, +} + +#[derive(Debug)] +struct ConsistentTxType; + +//Event processing depends on the marker type. +#[derive(Debug, Clone)] +pub struct EventTx { + pub content: EventTxContent, + pub tx_type: T, +} + +impl EventTx { + //helper method to transition from one event to another. + fn transition(&self, tx_type: N) -> EventTx { + EventTx { + content: self.content.clone(), + tx_type, + } + } +} + +//Processing of event that arrive: SourceTxType. +impl EventTx { + //Tx validation process. + async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { + let tx: Transaction = self.content.clone().into(); + tx.validate()?; + + // Secondly verify that author is whitelisted. + if !acl_whitelist.contains(&tx.author).await? { + return Err(TransactionError::Validation( + "Tx permission denied signer not authorized".to_string(), + )); + } + + if let Payload::Run { ref workflow } = self.content.payload { + let mut programs = HashSet::new(); + for step in &workflow.steps { + if !programs.insert(step.program) { + return Err(TransactionError::Validation(format!( + "multiple programs in workflow: {}", + &step.program + ))); + } + } + } + + Ok(()) + } + + fn verify_tx_signature(&self) -> bool { + let mut hasher = Sha3_256::new(); + let mut buf = vec![]; + hasher.update(self.content.author.serialize()); + self.content.payload.serialize_into(&mut buf); + hasher.update(buf); + hasher.update(self.content.nonce.to_be_bytes()); + + let hash: Hash = (&hasher.finalize()[0..32]).into(); + let msg: Message = hash.into(); + verify(&msg, &self.content.signature.into(), &self.content.author) + } + + fn propagate(self) -> Option { + match self.tx_type { + SourceTxType::P2P => None, + SourceTxType::RPC => Some(PropagateTx(EventTx { + content: self.content, + tx_type: ConsistentTxType, + })), + SourceTxType::TXRESULT => Some(PropagateTx(EventTx { + content: self.content, + tx_type: ConsistentTxType, + })), + } + } +} + +//RcvTx processing +pub struct RcvTx(pub EventTx); +impl RcvTx { + pub async fn process_event( + self, + acl_whitelist: &impl AclWhitelist, + ) -> Result { + match self.0.validate_tx(acl_whitelist).await { + Ok(()) => Ok(DownloadTx(self.0)), + Err(err) => Err(EventProcessError::ValidateError(format!( + "Tx validation fail:{err}" + ))), + } + } +} + +//Download Tx processing +pub struct DownloadTx(EventTx); +impl DownloadTx { + pub async fn process_event( + self, + local_directory_path: &PathBuf, + http_peer_list: Vec<(SocketAddr, Option)>, + ) -> Result<(NewTx, Option), EventProcessError> { + let tx_hash = self.0.content.hash; + let http_client = reqwest::Client::new(); + let asset_file_list = self + .0 + .content + .payload + .get_asset_list(tx_hash) + .map_err(|err| { + EventProcessError::DownloadAssetError(format!( + "Asset file param conversion error:{err}" + )) + })?; + + let futures: Vec<_> = asset_file_list + .into_iter() + .map(|asset_file| { + download_manager::download_asset_file( + local_directory_path, + &http_peer_list, + &http_client, + asset_file, + ) + }) + .collect(); + join_all(futures) + .await + .into_iter() + .map(|res| res) + .collect::, _>>() + .map_err(|err| { + EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) + })?; + let newtx = NewTx(self.0.transition(ConsistentTxType)); + let propagate = self.0.propagate(); + Ok((newtx, propagate)) + } +} + +//Propagate Tx processing +pub struct PropagateTx(EventTx); +impl PropagateTx { + pub async fn process_event( + self, + p2p_sender: &UnboundedSender, + ) -> Result<(), EventProcessError> { + let tx: Transaction = self.0.content.into(); + p2p_sender.send(tx).map_err(|err| err.into()) + } +} + +//Save new Tx processing +pub struct NewTx(EventTx); +impl NewTx { + pub async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { + let tx: Transaction = self.0.content.into(); + mempool + .add(tx) + .map_err(|err| EventProcessError::SaveTxError(format!("{err}"))) + .await + } +} diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs new file mode 100644 index 00000000..be4fe7df --- /dev/null +++ b/crates/node/src/txvalidation/mod.rs @@ -0,0 +1,198 @@ +use crate::txvalidation::event::{EventTx, RcvTx, SourceTxType}; +use crate::types::Transaction; +use crate::Mempool; +use futures_util::Stream; +use futures_util::TryFutureExt; +use gevulot_node::types::transaction::AclWhitelist; +use std::collections::HashMap; +use std::fmt::Debug; +use std::marker::PhantomData; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use thiserror::Error; +use tokio::sync::mpsc; +use tokio::sync::mpsc::UnboundedSender; +use tokio::sync::oneshot; +use tokio::sync::RwLock; +use tokio::task::JoinHandle; +use tokio_stream::wrappers::UnboundedReceiverStream; + +mod download_manager; +mod event; + +#[derive(Error, Debug)] +pub enum EventProcessError { + #[error("Fail to rcv Tx from the channel: {0}")] + RcvChannelError(#[from] tokio::sync::oneshot::error::RecvError), + #[error("Fail to send the Tx on the channel: {0}")] + SendChannelError(#[from] tokio::sync::mpsc::error::SendError<(RcvTx, Option)>), + #[error("Fail to send the Tx on the channel: {0}")] + PropagateTxError(#[from] tokio::sync::mpsc::error::SendError), + #[error("validation fail: {0}")] + ValidateError(String), + #[error("Tx asset fail to download because {0}")] + DownloadAssetError(String), + #[error("Save Tx error: {0}")] + SaveTxError(String), +} + +pub type CallbackSender = oneshot::Sender>; + +//Sending Tx interface. +//Some marker type to define the sender source. +pub struct RpcSender; +#[derive(Clone)] +pub struct P2pSender; +pub struct TxResultSender; + +//use to send a tx to the event process. +#[derive(Debug, Clone)] +pub struct TxEventSender { + sender: UnboundedSender<(RcvTx, Option)>, + _marker: PhantomData, +} + +//Manage send from the p2p source +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::P2P, + }; + self.sender + .send((RcvTx(event), None)) + .map_err(|err| err.into()) + } +} + +//Manage send from the RPC source +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let (sender, rx) = oneshot::channel(); + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::RPC, + }; + self.sender + .send((RcvTx(event), Some(sender))) + .map_err(|err| EventProcessError::from(err))?; + rx.await? + } +} + +//Manage send from the Tx result execution source +impl TxEventSender { + pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + TxEventSender { + sender, + _marker: PhantomData, + } + } + + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + let (sender, rx) = oneshot::channel(); + let event = EventTx { + content: tx.into(), + tx_type: SourceTxType::TXRESULT, + }; + self.sender + .send((RcvTx(event), Some(sender))) + .map_err(|err| EventProcessError::from(err))?; + rx.await? + } +} + +//Main event processing loog. +pub async fn start_event_loop( + local_directory_path: PathBuf, + bind_addr: SocketAddr, + http_download_port: u16, + http_peer_list: Arc>>>, + acl_whitelist: Arc, + //New Tx are added to the mempool directly. + //Like for the p2p a stream can be use to decouple both process. + mempool: Arc>, +) -> eyre::Result<( + JoinHandle<()>, + //channel use to send RcvTx event to the processing + UnboundedSender<(RcvTx, Option)>, + //output stream use to propagate Tx. + impl Stream, +)> { + let local_directory_path = Arc::new(local_directory_path); + //start http download manager + let download_jh = + download_manager::serve_files(bind_addr, http_download_port, local_directory_path.clone()) + .await?; + + let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel::<(RcvTx, Option)>(); + + let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::(); + let p2p_stream = UnboundedReceiverStream::new(p2p_recv); + let jh = tokio::spawn({ + let local_directory_path = local_directory_path.clone(); + + async move { + while let Some((event, callback)) = rcv_tx_event_rx.recv().await { + //process RcvTx(EventTx) event + let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; + + //process the receive event + tokio::spawn({ + let p2p_sender = p2p_sender.clone(); + let local_directory_path = local_directory_path.clone(); + let acl_whitelist = acl_whitelist.clone(); + let mempool = mempool.clone(); + async move { + let res = event + .process_event(acl_whitelist.as_ref()) + .and_then(|download_event| { + download_event.process_event(&local_directory_path, http_peer_list) + }) + .and_then(|(new_tx, propagate_tx)| async move { + if let Some(propagate_tx) = propagate_tx { + propagate_tx.process_event(&p2p_sender).await?; + } + new_tx.process_event(&mut *(mempool.write().await)).await?; + + Ok(()) + }) + .await; + //send the execution result back if needed. + if let Some(callback) = callback { + //forget the result because if the RPC connection is closed the send can fail. + let _ = callback.send(res); + } + } + }); + } + } + }); + Ok((jh, tx, p2p_stream)) +} + +async fn convert_peer_list_to_vec( + http_peer_list: &tokio::sync::RwLock>>, +) -> Vec<(SocketAddr, Option)> { + http_peer_list + .read() + .await + .iter() + .map(|(a, p)| (*a, *p)) + .collect() +} diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 9e76f1af..1918e226 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -1,71 +1,141 @@ use crate::types::transaction; use crate::types::Hash; +use eyre::Result; use serde::Deserialize; use serde::Serialize; use std::path::Path; use std::path::PathBuf; +//describe file data for a Tx #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] -pub struct AssetFile { - #[serde(skip_serializing, skip_deserializing)] - pub tx: Hash, +pub struct TxFile { pub name: String, pub url: String, + pub checksum: Hash, } -impl AssetFile { - pub fn get_relatif_path(&self) -> PathBuf { - //TODO use file name because the checksum is not saved in the Db. - //checksum is more secure to name collision. - let file_name = Path::new(&self.name).file_name().unwrap(); - PathBuf::new().join(self.tx.to_string()).join(file_name) - } - - pub fn try_from_prg_meta_data( - value: &transaction::ProgramMetadata, - tx_hash: Hash, - ) -> Result<(Self, Hash), &'static str> { - Ok(( - AssetFile { - url: value.image_file_url.clone(), - name: value.name.clone(), - tx: tx_hash, - }, - value.image_file_checksum.clone().into(), - )) - } - +impl TxFile { pub fn try_from_prg_data( value: &transaction::ProgramData, - tx_hash: Hash, - ) -> Result { + ) -> Result, &'static str> { let file = match value { transaction::ProgramData::Input { file_name, file_url, - .. - } => AssetFile { + checksum, + } => Some(TxFile { url: file_url.clone(), name: file_name.clone(), - tx: tx_hash, - }, + checksum: checksum.clone().into(), + }), transaction::ProgramData::Output { - source_program, - file_name, + source_program: _, + file_name: _, } => { - //pick from workflow::workflow_step_to_task - todo!() - // // Make record of file that needs transfer from source tx to current tx's files. - // file_transfers.push((*source_program, file_name.clone())); - - // AssetFile { - // tx, - // name: file_name.clone(), - // url: "".to_string(), - // } + //Output file are not use by Tx management. + None } }; Ok(file) } + + pub fn to_asset_file(self, tx_hash: Hash) -> AssetFile { + AssetFile::from_txfile(self, tx_hash) + } +} + +pub fn vec_txfile_to_bytes(vec: &[TxFile]) -> Result, bincode::Error> { + bincode::serialize(vec) +} + +//describe file data to calculate the file path for read/write +#[derive(Clone, Debug)] +pub struct AssetFile { + //true if the file is an image. + pub image: bool, + pub tx: Hash, + pub name: String, + pub url: String, + pub checksum: Hash, +} + +impl AssetFile { + fn from_txfile(file: TxFile, tx_hash: Hash) -> Self { + AssetFile { + image: false, + url: file.url, + name: file.name, + tx: tx_hash, + checksum: file.checksum, + } + } + + pub fn get_relatif_path(&self) -> PathBuf { + let file_name = Path::new(&self.name).file_name().unwrap(); + let mut path = if self.image { + PathBuf::from("images").join(self.tx.to_string()) + } else { + PathBuf::from(self.tx.to_string()) + }; + path.push(file_name); + path + } + + pub fn try_from_prg_meta_data( + value: &transaction::ProgramMetadata, + ) -> Result { + Ok(AssetFile { + image: true, + url: value.image_file_url.clone(), + name: value.name.clone(), + tx: value.hash, + checksum: value.image_file_checksum.clone().into(), + }) + } +} + +pub async fn move_vmfile(source: &VmFile, dest: &AssetFile, base_path: &Path) -> Result<()> { + let src_file_path = base_path.to_path_buf().join(&source.get_vmrelatif_path()); + let dst_file_path = base_path.to_path_buf().join(&dest.get_relatif_path()); + + tracing::debug!( + "moving file from {:#?} to {:#?}", + src_file_path, + dst_file_path + ); + + // Ensure any necessary subdirectories exists. + if let Some(parent) = dst_file_path.parent() { + tokio::fs::create_dir_all(parent) + .await + .expect("task file mkdir"); + } + + tokio::fs::rename(src_file_path, dst_file_path) + .await + .map_err(|e| e.into()) +} + +//describe file data to calculate the file path for read/write +#[derive(Clone, Debug)] +pub struct VmFile { + pub task_id: Hash, + pub name: String, +} + +impl VmFile { + pub fn get_vmrelatif_path(&self) -> PathBuf { + let name_path = Path::new(&self.name); + + //remove root from the path because PathBuf push replace when using abs path. + let file_path = if name_path.is_absolute() { + &self.name[1..] + } else { + &self.name[..] + }; + let mut path = PathBuf::from(self.task_id.to_string()); + path.push(file_path); + path + } } diff --git a/crates/node/src/types/task.rs b/crates/node/src/types/task.rs index 9c68b2d8..2ed37ad8 100644 --- a/crates/node/src/types/task.rs +++ b/crates/node/src/types/task.rs @@ -1,4 +1,4 @@ -use crate::types::file::AssetFile; +use crate::types::file::TxFile; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -37,7 +37,7 @@ pub struct Task { pub program_id: Hash, pub args: Vec, #[sqlx(skip)] - pub files: Vec, + pub files: Vec, #[serde(skip_deserializing)] pub serial: i32, #[serde(skip_deserializing)] diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index eee2d305..3346f09d 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -1,6 +1,8 @@ -use super::file::AssetFile; +use super::file::TxFile; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; +use crate::types::file::vec_txfile_to_bytes; +use crate::types::file::AssetFile; use crate::types::transaction; use async_trait::async_trait; use eyre::Result; @@ -8,7 +10,8 @@ use libsecp256k1::{sign, verify, Message, PublicKey, SecretKey}; use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use sha3::{Digest, Sha3_256}; -use std::{collections::HashSet, rc::Rc}; +use std::path::Path; +use std::{collections::HashSet, sync::Arc}; use thiserror::Error; #[async_trait] @@ -171,6 +174,7 @@ pub enum Payload { parent: Hash, prover: Hash, proof: Vec, + files: Vec, }, ProofKey { parent: Hash, @@ -180,6 +184,7 @@ pub enum Payload { parent: Hash, verifier: Hash, verification: Vec, + files: Vec, }, Cancel { parent: Hash, @@ -187,15 +192,15 @@ pub enum Payload { } impl Payload { - pub fn get_asset_list(&self, tx_hash: Hash) -> Result> { + pub fn get_asset_list(&self, tx_hash: Hash) -> Result> { match self { transaction::Payload::Deploy { prover, verifier, .. } => Ok(vec![ - AssetFile::try_from_prg_meta_data(prover, tx_hash).map_err(|err| { + AssetFile::try_from_prg_meta_data(prover).map_err(|err| { TransactionError::Validation(format!("Fail to get prover image file: {err}",)) })?, - AssetFile::try_from_prg_meta_data(verifier, tx_hash).map_err(|err| { + AssetFile::try_from_prg_meta_data(verifier).map_err(|err| { TransactionError::Validation( format!("Fail to gfet verifier image file: {err}",), ) @@ -223,28 +228,36 @@ impl Payload { .map(|(file_name, file_url, checksum)| { //verify the url is valide. reqwest::Url::parse(&file_url)?; - Ok(( - AssetFile { - url: file_url.clone(), - name: file_name.to_string(), - tx: tx_hash, - // checksum: checksum.to_string().into(), - }, - checksum.to_string().into(), - )) + Ok(AssetFile { + image: false, + url: file_url.clone(), + name: file_name.to_string(), + tx: tx_hash, + checksum: checksum.to_string().into(), + }) }) .collect() } - Payload::Proof { - parent, - prover, - proof, - } => todo!(), - Payload::Verification { - parent, - verifier, - verification, - } => todo!(), + Payload::Proof { files, .. } | Payload::Verification { files, .. } => files + .iter() + .map(|file| { + //get uuid from file name + let uuid = Path::new(&file.name).file_name().unwrap(); + + Ok(AssetFile { + image: false, + url: format!( + "{}/{}/{}", + file.url, + tx_hash.to_string(), + uuid.to_str().unwrap() + ), + name: file.name.to_string(), + tx: tx_hash, + checksum: file.checksum.to_string().into(), + }) + }) + .collect(), // Other transaction types don't have external assets that would // need processing. _ => Ok(vec![]), @@ -280,10 +293,13 @@ impl Payload { parent, prover, proof, + files, } => { buf.append(&mut parent.to_vec()); buf.append(&mut prover.to_vec()); buf.append(proof.clone().as_mut()); + buf.append(proof.clone().as_mut()); + buf.append(&mut vec_txfile_to_bytes(files).unwrap()); } Payload::ProofKey { parent, key } => { buf.append(&mut parent.to_vec()); @@ -293,10 +309,12 @@ impl Payload { parent, verifier, verification, + files, } => { buf.append(&mut parent.to_vec()); buf.append(&mut verifier.to_vec()); buf.append(verification.clone().as_mut()); + buf.append(&mut vec_txfile_to_bytes(files).unwrap()); } Payload::Cancel { parent } => { buf.append(&mut parent.to_vec()); diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 459ecbf7..157ba019 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -285,7 +285,7 @@ impl Provider for Qemu { match Qmp::new(format!("localhost:{qmp_port}")).await { Ok(c) => client = Some(c), - Err(_) => { + Err(err) => { retry_count += 1; sleep(Duration::from_millis(10)).await; } diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index 9e543b84..8032256d 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -313,6 +313,24 @@ impl VmService for VMServer { let result = request.into_inner().result; if let Some(result) = result { + if let task_result_request::Result::Task(ref result) = result { + // Save resulting files. + for file in result.files.clone() { + tracing::debug!("VM submit_result save file:{}", file.path); + if let Err(err) = self + .file_storage + .save_task_file(&result.id, &file.path, file.data) + .await + { + tracing::error!( + "failed to save task {} result file {}", + result.id, + file.path + ); + } + } + } + self.task_source.submit_result(program, vm_id, result).await; } diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index fce95494..9f82c253 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -1,17 +1,14 @@ -use crate::types::file::AssetFile; -use std::sync::Arc; - +use crate::types::file::TxFile; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::{ - transaction::{Payload, ProgramData, Workflow, WorkflowStep}, + transaction::{Payload, Workflow, WorkflowStep}, Hash, Task, TaskKind, Transaction, }; +use std::sync::Arc; use thiserror::Error; use uuid::Uuid; -use crate::storage; - #[allow(clippy::enum_variant_names)] #[derive(Error, Debug, PartialEq)] pub enum WorkflowError { @@ -38,14 +35,14 @@ pub trait TransactionStore: Sync + Send { pub struct WorkflowEngine { tx_store: Arc, - file_storage: Arc, + // file_storage: Arc, } impl WorkflowEngine { - pub fn new(tx_store: Arc, file_storage: Arc) -> Self { + pub fn new(tx_store: Arc) -> Self { WorkflowEngine { tx_store, - file_storage, + // file_storage, } } @@ -73,6 +70,7 @@ impl WorkflowEngine { parent, prover, proof, + .. } => { tracing::debug!("creating next task from Proof tx {}", &cur_tx.hash); @@ -123,6 +121,7 @@ impl WorkflowEngine { parent, prover, proof, + .. } = proof_tx.payload { match workflow.steps.iter().position(|s| s.program == prover) { @@ -271,13 +270,14 @@ impl WorkflowEngine { kind: TaskKind, ) -> Result { let id = Uuid::new_v4(); - let mut file_transfers: Vec<(Hash, String)> = vec![]; + let file_transfers: Vec<(Hash, String)> = vec![]; let files = step .inputs .iter() - .map(|e| { - AssetFile::try_from_prg_data(e, tx) + .filter_map(|e| { + TxFile::try_from_prg_data(e) .map_err(|err| WorkflowError::FileDefinitionError(err.to_string()).into()) + .transpose() }) // match e { // ProgramData::Input { @@ -304,18 +304,18 @@ impl WorkflowEngine { // }) .collect::>>()?; - // Process file transfers from source programs. - //TODO! move end task execution. - for (source_program, file_name) in file_transfers { - let source_tx = self - .find_parent_tx_for_program(&tx, &source_program) - .await - .expect("output file dependency missing"); - - self.file_storage - .move_task_file(&source_tx.to_string(), &tx.to_string(), &file_name) - .await?; - } + // // Process file transfers from source programs. + // //TODO! move end task execution. + // for (source_program, file_name) in file_transfers { + // let source_tx = self + // .find_parent_tx_for_program(&tx, &source_program) + // .await + // .expect("output file dependency missing"); + + // self.file_storage + // .move_task_file(&source_tx.to_string(), &tx.to_string(), &file_name) + // .await?; + // } Ok(Task { id, diff --git a/crates/shim/Cargo.toml b/crates/shim/Cargo.toml index f1cb7b83..a261c6d1 100644 --- a/crates/shim/Cargo.toml +++ b/crates/shim/Cargo.toml @@ -8,6 +8,7 @@ license = "MIT OR Apache-2.0" anyhow = "1" async-stream = "0.3.5" prost = "0.11" +sha3 = "0.10" tokio = { version = "1.0", features = ["fs", "macros", "rt-multi-thread"] } tokio-stream = "0.1" tokio-vsock = { version = "0.4.0", features = ["tonic-conn"] } diff --git a/crates/shim/src/lib.rs b/crates/shim/src/lib.rs index 217a0bbd..5db5ff4e 100644 --- a/crates/shim/src/lib.rs +++ b/crates/shim/src/lib.rs @@ -1,11 +1,11 @@ +use grpc::vm_service_client::VmServiceClient; +use grpc::{FileChunk, FileData, FileMetadata}; +use sha3::{Digest, Sha3_256}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; use std::time::Instant; use std::{path::Path, thread::sleep, time::Duration}; - -use grpc::vm_service_client::VmServiceClient; -use grpc::{FileChunk, FileData, FileMetadata}; use tokio::io::AsyncReadExt; use tokio::runtime::Runtime; use tokio::{io::AsyncWriteExt, sync::Mutex}; @@ -208,6 +208,25 @@ impl GRPCClient { id: result.id.clone(), data: result.data.clone(), files: result.files.clone(), + //TODO Add filecheckout. + // ======= + // files: result + // .files + // .iter() + // .map(|f| { + // let data = std::fs::read(f).expect("result file read"); + // let mut hasher = Sha3_256::new(); + // hasher.update(&data); + // let checksum = hasher.finalize(); + + // grpc::File { + // path: f.to_string(), + // data, + // checksum: checksum.to_vec(), + // } + // }) + // .collect(), + // >>>>>>> 0a0761b (correct file management. Work locally. Before remote test) })), }; From 005663065c9db5d1cad2a2ae7740f9a2d6f0659e Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 9 Feb 2024 10:30:15 +0100 Subject: [PATCH 03/43] correct p2p peer list init --- crates/node/src/main.rs | 28 ---------------- crates/node/src/networking/p2p/pea2pea.rs | 41 ++++++++++++++++++++++- 2 files changed, 40 insertions(+), 29 deletions(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 21f38570..e5368718 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -227,12 +227,6 @@ async fn run(config: Arc) -> Result<()> { .await, ); - // p2p.register_tx_handler(Arc::new(P2PTxHandler::new( - // mempool.clone(), - // database.clone(), - // ))) - // .await; - // TODO(tuommaki): read total available resources from config / acquire system stats. let num_gpus = if config.gpu_devices.is_some() { 1 } else { 0 }; let resource_manager = Arc::new(Mutex::new(scheduler::ResourceManager::new( @@ -252,20 +246,8 @@ async fn run(config: Arc) -> Result<()> { resource_manager.clone(), ); - // let asset_mgr = Arc::new(AssetManager::new( - // config.clone(), - // database.clone(), - // p2p.as_ref().peer_http_port_list.clone(), - // )); - let node_key = read_node_key(&config.node_key_file)?; - // Launch AssetManager's background processing. - // tokio::spawn({ - // let asset_mgr = asset_mgr.clone(); - // async move { asset_mgr.run().await } - // }); - let workflow_engine = Arc::new(WorkflowEngine::new(database.clone())); let download_url_prefix = format!( "http://{}:{}", @@ -342,16 +324,6 @@ async fn run(config: Arc) -> Result<()> { async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { let http_peer_list: Arc>>> = Default::default(); - // //start Tx process event loop - // let (txevent_loop_jh, tx_sender, p2p_stream) = start_event_loop( - // config.data_directory.clone(), - // config.p2p_listen_addr, - // config.http_download_port, - // http_peer_list.clone(), - // Arc::new(crate::types::transaction::AlwaysGrantAclWhitelist {}), - // mempool.clone(), - // ) - // .await?; //build empty channel for P2P interface Transaction management. //Indicate some domain conflict issue. diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index 5c82f5dd..b184e99b 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -103,7 +103,7 @@ impl P2P { psk: psk.to_vec(), peer_list: Default::default(), peer_addr_mapping: Default::default(), - peer_http_port_list: Default::default(), + peer_http_port_list, http_port, nat_listen_addr, tx_sender, @@ -498,6 +498,21 @@ mod tests { Arc::new(Sink::new(Arc::new(tx2))), Arc::new(Sink::new(Arc::new(tx3))), ); + let http_peer_list1: Arc>>> = + Default::default(); + let (_, p2p_recv1) = mpsc::unbounded_channel::(); + let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv); + + let http_peer_list2: Arc>>> = + Default::default(); + let (_, p2p_recv2) = mpsc::unbounded_channel::(); + let p2p_stream2 = UnboundedReceiverStream::new(p2p_recv); + + let http_peer_list3: Arc>>> = + Default::default(); + let (_, p2p_recv3) = mpsc::unbounded_channel::(); + let p2p_stream3 = UnboundedReceiverStream::new(p2p_recv); + let (peer1, peer2, peer3) = ( P2P::new( "peer1", @@ -505,6 +520,9 @@ mod tests { "secret passphrase", None, None, + http_peer_list1, + p2p_recv1, + p2p_stream1, ) .await, P2P::new( @@ -513,6 +531,9 @@ mod tests { "secret passphrase", Some(9995), None, + http_peer_list2, + p2p_recv2, + p2p_stream2, ) .await, P2P::new( @@ -521,6 +542,9 @@ mod tests { "secret passphrase", Some(9995), None, + http_peer_list3, + p2p_recv3, + p2p_stream3, ) .await, ); @@ -589,6 +613,15 @@ mod tests { Arc::new(Sink::new(Arc::new(tx1))), Arc::new(Sink::new(Arc::new(tx2))), ); + let http_peer_list1: Arc>>> = + Default::default(); + let (_, p2p_recv1) = mpsc::unbounded_channel::(); + let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv); + + let http_peer_list2: Arc>>> = + Default::default(); + let (_, p2p_recv2) = mpsc::unbounded_channel::(); + let p2p_stream2 = UnboundedReceiverStream::new(p2p_recv); let peer1 = P2P::new( "peer1", @@ -596,6 +629,9 @@ mod tests { "secret passphrase", None, None, + http_peer_list1, + http_peer_list1, + p2p_recv1, ) .await; peer1.node().start_listening().await.expect("peer1 listen"); @@ -608,6 +644,9 @@ mod tests { "secret passphrase", Some(8776), None, + http_peer_list2, + http_peer_list2, + p2p_recv2, ) .await; peer2.node().start_listening().await.expect("peer2 listen"); From 53d5423cfe0e26a422e8309146d0f2c488f60cfc Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 9 Feb 2024 11:18:39 +0100 Subject: [PATCH 04/43] add some logs to follow tx --- crates/node/src/main.rs | 32 ++--------------------- crates/node/src/networking/p2p/pea2pea.rs | 2 +- crates/node/src/scheduler/mod.rs | 3 +++ crates/node/src/txvalidation/event.rs | 2 ++ 4 files changed, 8 insertions(+), 31 deletions(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index e5368718..8f46509c 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -163,36 +163,6 @@ impl workflow::TransactionStore for storage::Database { } } -// struct P2PTxHandler { -// mempool: Arc>, -// database: Arc, -// } - -// impl P2PTxHandler { -// pub fn new(mempool: Arc>, database: Arc) -> Self { -// Self { mempool, database } -// } -// } - -// #[async_trait::async_trait] -// impl networking::p2p::TxHandler for P2PTxHandler { -// async fn recv_tx(&self, tx: Transaction) -> Result<()> { -// // The transaction was received from P2P network so we can consider it -// // propagated at this point. -// let tx_hash = tx.hash; -// let mut tx = tx; -// tx.propagated = true; - -// // Submit the tx to mempool. -// self.mempool.write().await.add(tx).await?; - -// //TODO copy paste of the asset manager handle_transaction method. -// //added because when a tx arrive from the p2p asset are not added. -// //should be done in a better way. -// // self.database.add_asset(&tx_hash).await -// } -// } - async fn run(config: Arc) -> Result<()> { let database = Arc::new(Database::new(&config.db_url).await?); @@ -255,6 +225,8 @@ async fn run(config: Arc) -> Result<()> { config.http_download_port ); + //TODO the exec result tx sender has to be added. + //TODO remove mempool. let scheduler = Arc::new(scheduler::Scheduler::new( mempool.clone(), database.clone(), diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index b184e99b..ba0a71bc 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -54,7 +54,7 @@ pub struct P2P { // This mapping is needed for proper cleanup on OnDisconnect. peer_addr_mapping: Arc>>, peer_list: Arc>>, - + //contains corrected peers use for asset file download. pub peer_http_port_list: Arc>>>, http_port: Option, diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index eb167dae..5d7f95d1 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -517,6 +517,8 @@ impl TaskManager for Scheduler { } }; + tracing::info!("Submit result Tx created:{}", tx.hash.to_string()); + //Move tx file from execution Tx path to new Tx path for (source_file, dest_file) in executed_files { let dest = dest_file.to_asset_file(tx.hash); @@ -527,6 +529,7 @@ impl TaskManager for Scheduler { } } + //TODO should be send to the validation process that verify, move the file and store it. let mut mempool = self.mempool.write().await; if let Err(err) = mempool.add(tx.clone()).await { tracing::error!("failed to add transaction to mempool: {}", err); diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index e127c144..611e7bef 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -212,6 +212,7 @@ impl PropagateTx { p2p_sender: &UnboundedSender, ) -> Result<(), EventProcessError> { let tx: Transaction = self.0.content.into(); + tracing::info!("Tx validation propagate tx:{}", tx.hash.to_string()); p2p_sender.send(tx).map_err(|err| err.into()) } } @@ -221,6 +222,7 @@ pub struct NewTx(EventTx); impl NewTx { pub async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { let tx: Transaction = self.0.content.into(); + tracing::info!("Tx validation save tx:{}", tx.hash.to_string()); mempool .add(tx) .map_err(|err| EventProcessError::SaveTxError(format!("{err}"))) From 30f9e488a3bbe4bee466b1a83c6f1ccc89c4e25f Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 9 Feb 2024 14:04:00 +0100 Subject: [PATCH 05/43] correct rebase error --- crates/node/src/rpc_server/mod.rs | 10 +------- crates/node/src/types/transaction.rs | 2 +- crates/node/src/vmm/vm_server.rs | 34 ++++++++++++++-------------- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index fd547c6d..74985189 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -1,5 +1,6 @@ use crate::txvalidation::RpcSender; use crate::txvalidation::TxEventSender; +use std::rc::Rc; use std::{net::SocketAddr, sync::Arc}; use crate::{cli::Config, storage::Database, types::Transaction}; @@ -83,15 +84,6 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp )); } - // Finally add it to mempool to propagate it to P2P network and wait - // to be scheduled for execution. - if let Err(err) = ctx.mempool.write().await.add(tx.clone()).await { - tracing::error!("failed to persist transaction: {}", err); - return RpcResponse::Err(RpcError::InvalidRequest( - "failed to persist transaction".to_string(), - )); - } - RpcResponse::Ok(()) } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 3346f09d..f3fecb40 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -11,7 +11,7 @@ use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use sha3::{Digest, Sha3_256}; use std::path::Path; -use std::{collections::HashSet, sync::Arc}; +use std::{collections::HashSet, rc::Rc}; use thiserror::Error; #[async_trait] diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index 8032256d..2af56c06 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -313,23 +313,23 @@ impl VmService for VMServer { let result = request.into_inner().result; if let Some(result) = result { - if let task_result_request::Result::Task(ref result) = result { - // Save resulting files. - for file in result.files.clone() { - tracing::debug!("VM submit_result save file:{}", file.path); - if let Err(err) = self - .file_storage - .save_task_file(&result.id, &file.path, file.data) - .await - { - tracing::error!( - "failed to save task {} result file {}", - result.id, - file.path - ); - } - } - } + // if let task_result_request::Result::Task(ref result) = result { + // // Save resulting files. + // for file in result.files.clone() { + // tracing::debug!("VM submit_result save file:{}", file.path); + // if let Err(err) = self + // .file_storage + // .save_task_file(&result.id, &file.path, file.data) + // .await + // { + // tracing::error!( + // "failed to save task {} result file {}", + // result.id, + // file.path + // ); + // } + // } + // } self.task_source.submit_result(program, vm_id, result).await; } From 557bc5873a8289579c63dbbb9fad8e4a1d7a68d3 Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 9 Feb 2024 16:16:16 +0100 Subject: [PATCH 06/43] add checksum to VM file and test --- crates/node/src/types/file.rs | 11 +++---- crates/node/src/types/hash.rs | 2 +- crates/node/src/vmm/vm_server.rs | 19 +---------- crates/shim/proto/vm_service.proto | 7 +++- crates/shim/src/lib.rs | 51 +++++++++++++++--------------- 5 files changed, 37 insertions(+), 53 deletions(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 1918e226..ec088db5 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -126,14 +126,11 @@ pub struct VmFile { impl VmFile { pub fn get_vmrelatif_path(&self) -> PathBuf { - let name_path = Path::new(&self.name); + let mut file_path = Path::new(&self.name); + if file_path.is_absolute() { + file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute + } - //remove root from the path because PathBuf push replace when using abs path. - let file_path = if name_path.is_absolute() { - &self.name[1..] - } else { - &self.name[..] - }; let mut path = PathBuf::from(self.task_id.to_string()); path.push(file_path); path diff --git a/crates/node/src/types/hash.rs b/crates/node/src/types/hash.rs index 92c24795..277f2b99 100644 --- a/crates/node/src/types/hash.rs +++ b/crates/node/src/types/hash.rs @@ -4,7 +4,7 @@ use serde::{de, Deserialize, Serialize}; use sqlx::{self, Decode, Encode, Postgres, Type}; use std::fmt; -const HASH_SIZE: usize = 32; +pub const HASH_SIZE: usize = 32; #[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq, Serialize, Deserialize)] pub struct Hash([u8; HASH_SIZE]); diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index 2af56c06..bc3175a4 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -226,6 +226,7 @@ impl VmService for VMServer { .join(self.file_storage.data_dir()) .join(task_id) .join(path); + tracing::trace!("VM submit_file saved in {file_path:#?}"); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { @@ -313,24 +314,6 @@ impl VmService for VMServer { let result = request.into_inner().result; if let Some(result) = result { - // if let task_result_request::Result::Task(ref result) = result { - // // Save resulting files. - // for file in result.files.clone() { - // tracing::debug!("VM submit_result save file:{}", file.path); - // if let Err(err) = self - // .file_storage - // .save_task_file(&result.id, &file.path, file.data) - // .await - // { - // tracing::error!( - // "failed to save task {} result file {}", - // result.id, - // file.path - // ); - // } - // } - // } - self.task_source.submit_result(program, vm_id, result).await; } diff --git a/crates/shim/proto/vm_service.proto b/crates/shim/proto/vm_service.proto index e575eb63..60d99b01 100644 --- a/crates/shim/proto/vm_service.proto +++ b/crates/shim/proto/vm_service.proto @@ -41,6 +41,11 @@ message GetFileRequest { string path = 2; } +message File { + string path = 1; + bytes checksum = 3; +} + message FileMetadata { string task_id = 1; string path = 2; @@ -65,7 +70,7 @@ message FileData { message TaskResult { string id = 1; bytes data = 2; - repeated string files = 3; + repeated File files = 3; } message TaskResultRequest { diff --git a/crates/shim/src/lib.rs b/crates/shim/src/lib.rs index 5db5ff4e..a693d777 100644 --- a/crates/shim/src/lib.rs +++ b/crates/shim/src/lib.rs @@ -4,6 +4,7 @@ use sha3::{Digest, Sha3_256}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; +use std::sync::Arc; use std::time::Instant; use std::{path::Path, thread::sleep, time::Duration}; use tokio::io::AsyncReadExt; @@ -154,9 +155,12 @@ impl GRPCClient { Ok(Some(task)) } - fn submit_file(&mut self, task_id: TaskId, file_path: String) -> Result<()> { + fn submit_file(&mut self, task_id: TaskId, file_path: String) -> Result> { + let hasher = Arc::new(Mutex::new(Sha3_256::new())); self.rt.block_on(async { + let stream_hasher = Arc::clone(&hasher); let outbound = async_stream::stream! { + let fd = match tokio::fs::File::open(&file_path).await { Ok(fd) => fd, Err(err) => { @@ -175,11 +179,12 @@ impl GRPCClient { yield metadata; let mut buf: [u8; DATA_STREAM_CHUNK_SIZE] = [0; DATA_STREAM_CHUNK_SIZE]; - loop { match file.read(&mut buf).await { Ok(0) => return, Ok(n) => { + + stream_hasher.lock().await.update(&buf[..n]); yield FileData{ result: Some(grpc::file_data::Result::Chunk(FileChunk{ data: buf[..n].to_vec() }))}; }, Err(err) => { @@ -194,39 +199,33 @@ impl GRPCClient { println!("failed to submit file: {}", err); } }); + let hasher = Arc::into_inner(hasher).unwrap().into_inner(); + let hash = hasher.finalize().to_vec(); - Ok(()) + Ok(hash) } fn submit_result(&mut self, result: &TaskResult) -> Result { - for file in &result.files { - self.submit_file(result.id.clone(), file.clone())?; - } + let files = result + .files + .iter() + .map(|file| { + self.submit_file(result.id.clone(), file.clone()) + .map(|checksum| crate::grpc::File { + path: file.to_string(), + checksum, + }) + }) + .collect::>>()?; + // for file in &result.files { + // let checksum = self.submit_file(result.id.clone(), file.clone())?; + // } let task_result_req = grpc::TaskResultRequest { result: Some(grpc::task_result_request::Result::Task(grpc::TaskResult { id: result.id.clone(), data: result.data.clone(), - files: result.files.clone(), - //TODO Add filecheckout. - // ======= - // files: result - // .files - // .iter() - // .map(|f| { - // let data = std::fs::read(f).expect("result file read"); - // let mut hasher = Sha3_256::new(); - // hasher.update(&data); - // let checksum = hasher.finalize(); - - // grpc::File { - // path: f.to_string(), - // data, - // checksum: checksum.to_vec(), - // } - // }) - // .collect(), - // >>>>>>> 0a0761b (correct file management. Work locally. Before remote test) + files: files, })), }; From 992de81cab0977112a7185e2c188a7de0245502e Mon Sep 17 00:00:00 2001 From: musitdev Date: Sun, 11 Feb 2024 22:00:23 +0100 Subject: [PATCH 07/43] add type state to file and Tx --- crates/node/src/asset_manager/mod.rs | 3 +- crates/node/src/main.rs | 16 +- crates/node/src/mempool/mod.rs | 35 ++- crates/node/src/networking/p2p/mod.rs | 2 +- crates/node/src/networking/p2p/pea2pea.rs | 82 +++--- crates/node/src/networking/p2p/protocol.rs | 2 +- crates/node/src/rpc_client/mod.rs | 12 +- crates/node/src/rpc_server/mod.rs | 16 +- crates/node/src/scheduler/mod.rs | 57 +++-- .../storage/database/entity/transaction.rs | 9 +- crates/node/src/storage/database/postgres.rs | 35 ++- .../node/src/txvalidation/download_manager.rs | 7 +- crates/node/src/txvalidation/event.rs | 234 +++++++----------- crates/node/src/txvalidation/mod.rs | 66 ++--- crates/node/src/types/file.rs | 195 ++++++++++----- crates/node/src/types/task.rs | 4 +- crates/node/src/types/transaction.rs | 165 ++++++------ crates/node/src/workflow/mod.rs | 10 +- crates/tests/e2e-tests/src/main.rs | 15 +- 19 files changed, 535 insertions(+), 430 deletions(-) diff --git a/crates/node/src/asset_manager/mod.rs b/crates/node/src/asset_manager/mod.rs index 66bc49c3..c2e2675c 100644 --- a/crates/node/src/asset_manager/mod.rs +++ b/crates/node/src/asset_manager/mod.rs @@ -5,6 +5,7 @@ use crate::{ types::{transaction::Transaction, Hash}, }; use eyre::Result; +use gevulot_node::types::transaction::TxValdiated; use std::collections::HashMap; use std::net::SocketAddr; use std::{sync::Arc, time::Duration}; @@ -77,7 +78,7 @@ impl AssetManagerOld { /// handle_transaction admits transaction into `AssetManager` for further /// processing. - pub async fn handle_transaction(&self, tx: &Transaction) -> Result<()> { + pub async fn handle_transaction(&self, tx: &Transaction) -> Result<()> { self.database.add_asset(&tx.hash).await } diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 8f46509c..ddae8149 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -28,7 +28,7 @@ use tokio::sync::{Mutex as TMutex, RwLock}; use tokio_stream::wrappers::UnboundedReceiverStream; use tonic::transport::Server; use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan, EnvFilter}; -use types::{Hash, Transaction}; +use types::{transaction::TxValdiated, Hash, Transaction}; use workflow::WorkflowEngine; mod asset_manager; @@ -136,18 +136,21 @@ fn generate_node_key(opts: NodeKeyOptions) -> Result<()> { #[async_trait] impl mempool::Storage for storage::Database { - async fn get(&self, hash: &Hash) -> Result> { + async fn get(&self, hash: &Hash) -> Result>> { self.find_transaction(hash).await } - async fn set(&self, tx: &Transaction) -> Result<()> { + async fn set(&self, tx: &Transaction) -> Result<()> { let tx_hash = tx.hash; self.add_transaction(tx).await?; self.add_asset(&tx_hash).await?; self.mark_asset_complete(&tx_hash).await } - async fn fill_deque(&self, deque: &mut std::collections::VecDeque) -> Result<()> { + async fn fill_deque( + &self, + deque: &mut std::collections::VecDeque>, + ) -> Result<()> { for t in self.get_unexecuted_transactions().await? { deque.push_back(t); } @@ -158,7 +161,7 @@ impl mempool::Storage for storage::Database { #[async_trait] impl workflow::TransactionStore for storage::Database { - async fn find_transaction(&self, tx_hash: &Hash) -> Result> { + async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { self.find_transaction(tx_hash).await } } @@ -235,6 +238,7 @@ async fn run(config: Arc) -> Result<()> { node_key, config.data_directory.clone(), download_url_prefix, + txvalidation::TxEventSender::::build(tx_sender.clone()), )); let file_storage = Arc::new(storage::File::new(&config.data_directory)); @@ -303,7 +307,7 @@ async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel(); tokio::spawn(async move { while let Some(_) = rcv_tx_event_rx.recv().await {} }); - let (_, p2p_recv) = mpsc::unbounded_channel::(); + let (_, p2p_recv) = mpsc::unbounded_channel::>(); let p2p_stream = UnboundedReceiverStream::new(p2p_recv); let p2p = Arc::new( diff --git a/crates/node/src/mempool/mod.rs b/crates/node/src/mempool/mod.rs index a458e927..35d9c134 100644 --- a/crates/node/src/mempool/mod.rs +++ b/crates/node/src/mempool/mod.rs @@ -1,20 +1,15 @@ +use crate::types::{transaction::TxValdiated, Hash, Transaction}; use async_trait::async_trait; use eyre::Result; use std::collections::VecDeque; use std::sync::Arc; use thiserror::Error; -use tokio::sync::RwLock; - -use crate::{ - networking, - types::{Hash, Transaction}, -}; #[async_trait] pub trait Storage: Send + Sync { - async fn get(&self, hash: &Hash) -> Result>; - async fn set(&self, tx: &Transaction) -> Result<()>; - async fn fill_deque(&self, deque: &mut VecDeque) -> Result<()>; + async fn get(&self, hash: &Hash) -> Result>>; + async fn set(&self, tx: &Transaction) -> Result<()>; + async fn fill_deque(&self, deque: &mut VecDeque>) -> Result<()>; } #[allow(clippy::enum_variant_names)] @@ -30,7 +25,7 @@ pub struct Mempool { // acl_whitelist: Arc, // TODO: This should be refactored to PubSub channel abstraction later on. // tx_chan: Option>, - deque: VecDeque, + deque: VecDeque>, } impl Mempool { @@ -50,16 +45,16 @@ impl Mempool { }) } - pub fn next(&mut self) -> Option { + pub fn next(&mut self) -> Option> { // TODO(tuommaki): Should storage reflect the POP in state? self.deque.pop_front() } - pub fn peek(&self) -> Option<&Transaction> { + pub fn peek(&self) -> Option<&Transaction> { self.deque.front() } - pub async fn add(&mut self, tx: Transaction) -> Result<()> { + pub async fn add(&mut self, tx: Transaction) -> Result<()> { self.storage.set(&tx).await?; self.deque.push_back(tx); Ok(()) @@ -70,11 +65,11 @@ impl Mempool { } } -pub struct P2PTxHandler(Arc>); +// pub struct P2PTxHandler(Arc>); -#[async_trait::async_trait] -impl networking::p2p::TxHandler for P2PTxHandler { - async fn recv_tx(&self, tx: Transaction) -> Result<()> { - self.0.write().await.add(tx).await - } -} +// #[async_trait::async_trait] +// impl networking::p2p::TxHandler for P2PTxHandler { +// async fn recv_tx(&self, tx: Transaction) -> Result<()> { +// self.0.write().await.add(tx).await +// } +// } diff --git a/crates/node/src/networking/p2p/mod.rs b/crates/node/src/networking/p2p/mod.rs index 7afa5bc3..f0fe97f1 100644 --- a/crates/node/src/networking/p2p/mod.rs +++ b/crates/node/src/networking/p2p/mod.rs @@ -1,4 +1,4 @@ mod noise; mod pea2pea; mod protocol; -pub use pea2pea::{TxHandler, P2P}; +pub use pea2pea::P2P; diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index ba0a71bc..0d22ba96 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -15,8 +15,10 @@ use tokio_stream::StreamExt; use super::{noise, protocol}; use bytes::{Bytes, BytesMut}; -use eyre::Result; -use gevulot_node::types::Transaction; +use gevulot_node::types::{ + transaction::{TxCreate, TxValdiated}, + Transaction, +}; use parking_lot::RwLock; use pea2pea::{ protocols::{Handshake, OnDisconnect, Reading, Writing}, @@ -24,31 +26,31 @@ use pea2pea::{ }; use sha3::{Digest, Sha3_256}; -#[async_trait::async_trait] -pub trait TxHandler: Send + Sync { - async fn recv_tx(&self, tx: Transaction) -> Result<()>; -} - -#[async_trait::async_trait] -pub trait TxChannel: Send + Sync { - async fn send_tx(&self, tx: &Transaction) -> Result<()>; -} - -struct BlackholeTxHandler; -#[async_trait::async_trait] -impl TxHandler for BlackholeTxHandler { - async fn recv_tx(&self, tx: Transaction) -> Result<()> { - tracing::debug!("submitting received tx to black hole"); - Ok(()) - } -} +// #[async_trait::async_trait] +// pub trait TxHandler: Send + Sync { +// async fn recv_tx(&self, tx: Transaction) -> Result<()>; +// } + +// #[async_trait::async_trait] +// pub trait TxChannel: Send + Sync { +// async fn send_tx(&self, tx: &Transaction) -> Result<()>; +// } + +// struct BlackholeTxHandler; +// #[async_trait::async_trait] +// impl TxHandler for BlackholeTxHandler { +// async fn recv_tx(&self, tx: Transaction) -> Result<()> { +// tracing::debug!("submitting received tx to black hole"); +// Ok(()) +// } +// } // NOTE: This P2P implementation is originally from `pea2pea` Noise handshake example. #[derive(Clone)] pub struct P2P { node: Node, noise_states: Arc>>, - tx_handler: Arc>>, + //tx_handler: Arc>>, // Peer connection map: <(P2P TCP connection's peer address) , (peer's advertised address in peer_list)>. // This mapping is needed for proper cleanup on OnDisconnect. @@ -80,7 +82,7 @@ impl P2P { nat_listen_addr: Option, peer_http_port_list: Arc>>>, tx_sender: TxEventSender, - propagate_tx_stream: impl Stream + std::marker::Send + 'static, + propagate_tx_stream: impl Stream> + std::marker::Send + 'static, ) -> Self { let config = Config { name: Some(name.into()), @@ -99,7 +101,7 @@ impl P2P { let instance = Self { node, noise_states: Default::default(), - tx_handler: Arc::new(tokio::sync::RwLock::new(Arc::new(BlackholeTxHandler {}))), + //tx_handler: Arc::new(tokio::sync::RwLock::new(Arc::new(BlackholeTxHandler {}))), psk: psk.to_vec(), peer_list: Default::default(), peer_addr_mapping: Default::default(), @@ -150,7 +152,7 @@ impl P2P { // tracing::debug!("new tx handler registered"); // } - async fn recv_tx(&self, tx: Transaction) { + async fn recv_tx(&self, tx: Transaction) { tracing::debug!("submitting received tx to tx_handler"); if let Err(err) = self.tx_sender.send_tx(tx) { tracing::error!("P2P error during received Tx sending:{err}"); @@ -408,6 +410,16 @@ impl Reading for P2P { tx.hash, hex::encode(tx.author.serialize()) ); + let tx: Transaction = Transaction { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + propagated: tx.propagated, + executed: tx.executed, + state: TxCreate, + }; self.recv_tx(tx).await; } protocol::MessageV0::DiagnosticsRequest(kind) => { @@ -424,17 +436,17 @@ impl Reading for P2P { } } -#[async_trait::async_trait] -impl TxChannel for P2P { - async fn send_tx(&self, tx: &Transaction) -> Result<()> { - let msg = protocol::Message::V0(protocol::MessageV0::Transaction(tx.clone())); - let bs = Bytes::from(bincode::serialize(&msg)?); - - tracing::debug!("broadcasting transaction {}", tx.hash); - self.broadcast(bs)?; - Ok(()) - } -} +// #[async_trait::async_trait] +// impl TxChannel for P2P { +// async fn send_tx(&self, tx: &Transaction) -> Result<()> { +// let msg = protocol::Message::V0(protocol::MessageV0::Transaction(tx.clone())); +// let bs = Bytes::from(bincode::serialize(&msg)?); + +// tracing::debug!("broadcasting transaction {}", tx.hash); +// self.broadcast(bs)?; +// Ok(()) +// } +// } impl Writing for P2P { type Message = Bytes; type Codec = noise::Codec; diff --git a/crates/node/src/networking/p2p/protocol.rs b/crates/node/src/networking/p2p/protocol.rs index 1675501b..0dae6b68 100644 --- a/crates/node/src/networking/p2p/protocol.rs +++ b/crates/node/src/networking/p2p/protocol.rs @@ -23,7 +23,7 @@ pub(crate) enum Message { #[allow(clippy::large_enum_variant)] #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) enum MessageV0 { - Transaction(types::Transaction), + Transaction(types::Transaction), DiagnosticsRequest(DiagnosticsRequestKind), DiagnosticsResponse(DiagnosticsResponseV0), } diff --git a/crates/node/src/rpc_client/mod.rs b/crates/node/src/rpc_client/mod.rs index eaffb24b..de79e03a 100644 --- a/crates/node/src/rpc_client/mod.rs +++ b/crates/node/src/rpc_client/mod.rs @@ -5,7 +5,11 @@ use jsonrpsee::{ http_client::{HttpClient, HttpClientBuilder}, }; -use crate::types::{rpc::RpcResponse, transaction::TransactionTree, Hash, Transaction}; +use crate::types::{ + rpc::RpcResponse, + transaction::{TransactionTree, TxCreate, TxValdiated}, + Hash, Transaction, +}; pub struct RpcClient { client: HttpClient, @@ -22,13 +26,13 @@ impl RpcClient { pub async fn get_transaction( &self, tx_hash: &Hash, - ) -> Result, Box> { + ) -> Result>, Box> { let mut params = ArrayParams::new(); params.insert(tx_hash).expect("rpc params"); let resp = self .client - .request::, ArrayParams>("getTransaction", params) + .request::>, ArrayParams>("getTransaction", params) .await .expect("rpc request"); @@ -38,7 +42,7 @@ impl RpcClient { } } - pub async fn send_transaction(&self, tx: &Transaction) -> Result<(), Box> { + pub async fn send_transaction(&self, tx: &Transaction) -> Result<(), Box> { let mut params = ArrayParams::new(); params.insert(tx).expect("rpc params"); diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index 74985189..3a6c192d 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -3,7 +3,14 @@ use crate::txvalidation::TxEventSender; use std::rc::Rc; use std::{net::SocketAddr, sync::Arc}; -use crate::{cli::Config, storage::Database, types::Transaction}; +use crate::{ + cli::Config, + storage::Database, + types::{ + transaction::{TxCreate, TxValdiated}, + Transaction, + }, +}; use eyre::Result; use gevulot_node::types::{ rpc::{RpcError, RpcResponse}, @@ -69,7 +76,7 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp tracing::info!("JSON-RPC: send_transaction()"); // Real logic - let tx: Transaction = match params.one() { + let tx: Transaction = match params.one() { Ok(tx) => tx, Err(e) => { tracing::error!("failed to parse transaction: {}", e); @@ -88,7 +95,10 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp } #[tracing::instrument(level = "info")] -async fn get_transaction(params: Params<'static>, ctx: Arc) -> RpcResponse { +async fn get_transaction( + params: Params<'static>, + ctx: Arc, +) -> RpcResponse> { let tx_hash: Hash = match params.one() { Ok(tx_hash) => tx_hash, Err(e) => { diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 5d7f95d1..b6534b62 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -2,7 +2,9 @@ mod program_manager; mod resource_manager; use crate::storage::Database; -use crate::types::file::{move_vmfile, TxFile, VmFile}; +use crate::txvalidation::TxEventSender; +use crate::txvalidation::TxResultSender; +use crate::types::file::{move_vmfile, File, ProofVerif, Vm}; use crate::types::TaskState; use crate::vmm::vm_server::grpc; use crate::vmm::{vm_server::TaskManager, VMId}; @@ -68,6 +70,7 @@ pub struct Scheduler { task_queue: Arc>>>, data_directory: PathBuf, http_download_host: String, + tx_sender: TxEventSender, } impl Scheduler { @@ -79,6 +82,7 @@ impl Scheduler { node_key: SecretKey, data_directory: PathBuf, http_download_host: String, + tx_sender: TxEventSender, ) -> Self { Self { mempool, @@ -93,6 +97,7 @@ impl Scheduler { task_queue: Arc::new(Mutex::new(HashMap::new())), data_directory, http_download_host, + tx_sender, } } @@ -417,30 +422,30 @@ impl TaskManager for Scheduler { } //move and build output files of the Tx execution - let executed_files: Vec<(VmFile, TxFile)> = result + let executed_files: Vec<(File, File)> = result .files - .iter() + .into_iter() .map(|file| { - let vm_file = VmFile { - task_id: running_task.task.tx, - name: file.path.to_string(), - }; + let vm_file = File::::new( + file.path.to_string(), + file.checksum[..].into(), + running_task.task.tx, + ); let uuid = Uuid::new_v4(); //format file_name to keep the current filename and the uuid. //put uuid at the end so that the uuid is use to save the file. let new_file_name = format!("{}/{}", file.path, uuid); - let dest = TxFile { - //Real url will be calculated during download. - url: self.http_download_host.clone(), - name: new_file_name, - checksum: file.checksum[..].into(), - }; + let dest = File::::new( + file.path, + self.http_download_host.clone(), + file.checksum[..].into(), + ); (vm_file, dest) }) .collect(); - //Verify that all expected files has been generated. + //TODO ? -Verify that all expected files has been generated. // let executed_files: Vec<(VmFile, TxFile)> = // if let Ok(Some(tx)) = self.database.find_transaction(&running_task.task.tx).await { // if let Payload::Run { workflow } = tx.payload { @@ -481,13 +486,13 @@ impl TaskManager for Scheduler { // vec![] // }; - let new_tx_files: Vec = executed_files + let new_tx_files: Vec> = executed_files .iter() .map(|(_, file)| file) .cloned() .collect(); let nonce = rand::thread_rng().next_u64(); - let tx = match running_task.task.kind { + let mut tx = match running_task.task.kind { TaskKind::Proof => Transaction::new( Payload::Proof { parent: running_task.task.tx, @@ -517,25 +522,27 @@ impl TaskManager for Scheduler { } }; + //TODO Add tx execution in the state need some Higher Kind type. + tx.executed = true; + tracing::info!("Submit result Tx created:{}", tx.hash.to_string()); //Move tx file from execution Tx path to new Tx path for (source_file, dest_file) in executed_files { - let dest = dest_file.to_asset_file(tx.hash); - if let Err(err) = move_vmfile(&source_file, &dest, &self.data_directory).await { + if let Err(err) = + move_vmfile(&source_file, &dest_file, &self.data_directory, tx.hash).await + { tracing::error!( - "failed to move excution file from: {source_file:?} to: {dest:?} error: {err}", + "failed to move excution file from: {source_file:?} to: {dest_file:?} error: {err}", ); } } - //TODO should be send to the validation process that verify, move the file and store it. - let mut mempool = self.mempool.write().await; - if let Err(err) = mempool.add(tx.clone()).await { - tracing::error!("failed to add transaction to mempool: {}", err); + //send tx to validation process. + if let Err(err) = self.tx_sender.send_tx(tx).await { + tracing::error!("failed to send Tx result to validation process: {}", err); } else { - dbg!(tx); - tracing::info!("successfully added new tx to mempool from task result"); + tracing::info!("successfully sent new tx to tx validation from task result"); } tracing::debug!("terminating VM {} running program {}", vm_id, program); diff --git a/crates/node/src/storage/database/entity/transaction.rs b/crates/node/src/storage/database/entity/transaction.rs index eeb4b6b5..4d6ef8a4 100644 --- a/crates/node/src/storage/database/entity/transaction.rs +++ b/crates/node/src/storage/database/entity/transaction.rs @@ -29,8 +29,8 @@ pub struct Transaction { pub executed: bool, } -impl From<&types::Transaction> for Transaction { - fn from(value: &types::Transaction) -> Self { +impl From<&types::Transaction> for Transaction { + fn from(value: &types::Transaction) -> Self { let kind = match value.payload { transaction::Payload::Empty => Kind::Empty, transaction::Payload::Transfer { .. } => Kind::Transfer, @@ -56,8 +56,8 @@ impl From<&types::Transaction> for Transaction { } } -impl From for types::Transaction { - fn from(value: Transaction) -> types::Transaction { +impl From for types::Transaction { + fn from(value: Transaction) -> types::Transaction { types::Transaction { author: value.author.into(), hash: value.hash, @@ -68,6 +68,7 @@ impl From for types::Transaction { signature: value.signature, propagated: value.propagated, executed: value.executed, + state: transaction::TxValdiated, } } } diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index c2a87a4e..8da188a7 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -1,7 +1,11 @@ use super::entity::{self}; -use crate::types::file::TxFile; +use crate::types::file::DbFile; use crate::types::transaction; -use crate::types::{self, transaction::ProgramData, Hash, Program, Task}; +use crate::types::{ + self, + transaction::{ProgramData, TxValdiated}, + Hash, Program, Task, +}; use eyre::Result; use gevulot_node::types::program::ResourceRequest; use gevulot_node::types::transaction::TransactionError; @@ -180,7 +184,7 @@ impl Database { match task { Some(mut task) => { let mut files = - sqlx::query_as::<_, TxFile>("SELECT * FROM file WHERE task_id = $1") + sqlx::query_as::<_, DbFile>("SELECT * FROM file WHERE task_id = $1") .bind(id) .fetch_all(&mut *tx) .await?; @@ -200,7 +204,7 @@ impl Database { .await?; for task in &mut tasks { - let mut files = sqlx::query_as::<_, TxFile>("SELECT * FROM file WHERE task_id = $1") + let mut files = sqlx::query_as::<_, DbFile>("SELECT * FROM file WHERE task_id = $1") .bind(task.id) .fetch_all(&mut *tx) .await?; @@ -263,7 +267,10 @@ impl Database { // transaction related operations. They are implemented naively on purpose // for now to maintain initial flexibility in development. Later on, these // queries here are easy low hanging fruits for optimizations. - pub async fn find_transaction(&self, tx_hash: &Hash) -> Result> { + pub async fn find_transaction( + &self, + tx_hash: &Hash, + ) -> Result>> { let mut db_tx = self.pool.begin().await?; let entity = @@ -364,7 +371,7 @@ impl Database { entity::transaction::Kind::Proof => { //get payload files let files = - sqlx::query_as::<_, TxFile>("SELECT * FROM txfile WHERE tx_id = $1") + sqlx::query_as::<_, DbFile>("SELECT * FROM txfile WHERE tx_id = $1") .bind(tx_hash) .fetch_all(&mut *db_tx) .await?; @@ -376,7 +383,7 @@ impl Database { parent: row.get(0), prover: row.get(1), proof: row.get(2), - files: files.clone(), //to avoid file move. + files: files.clone().into_iter().map(|file| file.into()).collect(), }, ) .fetch_one(&mut *db_tx) @@ -397,7 +404,7 @@ impl Database { entity::transaction::Kind::Verification => { //get payload files let files = - sqlx::query_as::<_, TxFile>("SELECT * FROM txfile WHERE tx_id = $1") + sqlx::query_as::<_, DbFile>("SELECT * FROM txfile WHERE tx_id = $1") .bind(tx_hash) .fetch_all(&mut *db_tx) .await?; @@ -410,7 +417,7 @@ impl Database { parent: row.get(0), verifier: row.get(1), verification: row.get(2), - files: files.clone(), //to avoid file move. + files: files.clone().into_iter().map(|file| file.into()).collect(), }, ) .fetch_one(&mut *db_tx) @@ -419,7 +426,7 @@ impl Database { _ => types::transaction::Payload::Empty, }; - let mut tx: types::transaction::Transaction = entity.into(); + let mut tx: types::transaction::Transaction = entity.into(); tx.payload = payload; Ok(Some(tx)) } else { @@ -427,7 +434,7 @@ impl Database { } } - pub async fn get_transactions(&self) -> Result> { + pub async fn get_transactions(&self) -> Result>> { let mut db_tx = self.pool.begin().await?; let refs: Vec = sqlx::query("SELECT hash FROM transaction") .map(|row: sqlx::postgres::PgRow| row.get(0)) @@ -445,7 +452,9 @@ impl Database { Ok(txs) } - pub async fn get_unexecuted_transactions(&self) -> Result> { + pub async fn get_unexecuted_transactions( + &self, + ) -> Result>> { let mut db_tx = self.pool.begin().await?; let refs: Vec = sqlx::query("SELECT hash FROM transaction WHERE executed IS false") .map(|row: sqlx::postgres::PgRow| row.get(0)) @@ -479,7 +488,7 @@ impl Database { Ok(refs) } - pub async fn add_transaction(&self, tx: &types::Transaction) -> Result<()> { + pub async fn add_transaction(&self, tx: &types::Transaction) -> Result<()> { let entity = entity::Transaction::from(tx); let mut db_tx = self.pool.begin().await?; diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 06f7e358..41afaad5 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -1,4 +1,4 @@ -use crate::types::file::AssetFile; +use crate::types::file::{Download, File}; use eyre::eyre; use eyre::Result; use futures_util::TryStreamExt; @@ -13,7 +13,6 @@ use std::net::SocketAddr; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use tokio::fs::File; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::task::JoinHandle; @@ -26,7 +25,7 @@ pub async fn download_asset_file( // file: &str, http_peer_list: &[(SocketAddr, Option)], http_client: &reqwest::Client, - asset_file: AssetFile, + asset_file: File, ) -> Result<()> { let local_relative_file_path = asset_file.get_relatif_path(); tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); @@ -162,7 +161,7 @@ async fn server_process_file( let mut file_path = data_directory.join(file_digest); - let file = match File::open(&file_path).await { + let file = match tokio::fs::File::open(&file_path).await { Ok(file) => file, Err(_) => { //try to see if the file is currently being updated. diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index 611e7bef..6277ecab 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -1,183 +1,124 @@ use crate::txvalidation::download_manager; use crate::txvalidation::EventProcessError; -use crate::types::transaction::Payload; -use crate::types::transaction::TransactionError; -use crate::types::Hash; -use crate::types::Signature; -use crate::types::Transaction; +use crate::types::{ + transaction::{TransactionError, TxReceive, TxValdiated}, + Transaction, +}; use crate::Mempool; use futures::future::join_all; use futures_util::TryFutureExt; use gevulot_node::types::transaction::AclWhitelist; -use libsecp256k1::verify; -use libsecp256k1::Message; -use libsecp256k1::PublicKey; -use sha3::{Digest, Sha3_256}; -use std::collections::HashSet; use std::fmt::Debug; use std::net::SocketAddr; use std::path::PathBuf; use tokio::sync::mpsc::UnboundedSender; -///Content of all Tx. +//event type. #[derive(Debug, Clone)] -pub struct EventTxContent { - pub author: PublicKey, - pub hash: Hash, - pub payload: Payload, - pub nonce: u64, - pub signature: Signature, -} - -impl From for EventTxContent { - fn from(tx: Transaction) -> Self { - EventTxContent { - author: tx.author, - hash: tx.hash, - payload: tx.payload, - nonce: tx.nonce, - signature: tx.signature, - } - } -} - -impl From for Transaction { - fn from(content: EventTxContent) -> Self { - Transaction { - author: content.author, - hash: content.hash, - payload: content.payload, - nonce: content.nonce, - signature: content.signature, - propagated: true, - executed: false, - } - } -} +pub struct RcvTx; -//event base type. +#[derive(Debug, Clone)] +pub struct DownloadTx; -//Marker type to define the event. -#[derive(Debug)] -pub enum SourceTxType { - P2P, - RPC, - TXRESULT, -} +#[derive(Debug, Clone)] +pub struct NewTx; -#[derive(Debug)] -struct ConsistentTxType; +#[derive(Debug, Clone)] +pub struct PropagateTx; //Event processing depends on the marker type. #[derive(Debug, Clone)] pub struct EventTx { - pub content: EventTxContent, + pub tx: Transaction, pub tx_type: T, } -impl EventTx { - //helper method to transition from one event to another. - fn transition(&self, tx_type: N) -> EventTx { +impl From> for EventTx { + fn from(tx: Transaction) -> Self { EventTx { - content: self.content.clone(), - tx_type, + tx: tx, + tx_type: RcvTx, } } } -//Processing of event that arrive: SourceTxType. -impl EventTx { - //Tx validation process. - async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { - let tx: Transaction = self.content.clone().into(); - tx.validate()?; - - // Secondly verify that author is whitelisted. - if !acl_whitelist.contains(&tx.author).await? { - return Err(TransactionError::Validation( - "Tx permission denied signer not authorized".to_string(), - )); - } - - if let Payload::Run { ref workflow } = self.content.payload { - let mut programs = HashSet::new(); - for step in &workflow.steps { - if !programs.insert(step.program) { - return Err(TransactionError::Validation(format!( - "multiple programs in workflow: {}", - &step.program - ))); - } - } +impl From> for EventTx { + fn from(event: EventTx) -> Self { + EventTx { + tx: event.tx, + tx_type: DownloadTx, } - - Ok(()) } +} - fn verify_tx_signature(&self) -> bool { - let mut hasher = Sha3_256::new(); - let mut buf = vec![]; - hasher.update(self.content.author.serialize()); - self.content.payload.serialize_into(&mut buf); - hasher.update(buf); - hasher.update(self.content.nonce.to_be_bytes()); - - let hash: Hash = (&hasher.finalize()[0..32]).into(); - let msg: Message = hash.into(); - verify(&msg, &self.content.signature.into(), &self.content.author) +impl From> for EventTx { + fn from(event: EventTx) -> Self { + EventTx { + tx: event.tx, + tx_type: NewTx, + } } +} - fn propagate(self) -> Option { - match self.tx_type { - SourceTxType::P2P => None, - SourceTxType::RPC => Some(PropagateTx(EventTx { - content: self.content, - tx_type: ConsistentTxType, - })), - SourceTxType::TXRESULT => Some(PropagateTx(EventTx { - content: self.content, - tx_type: ConsistentTxType, - })), +impl From> for Option> { + fn from(event: EventTx) -> Self { + match event.tx.state { + TxReceive::P2P => None, + TxReceive::RPC => Some(EventTx { + tx: event.tx, + tx_type: PropagateTx, + }), + TxReceive::TXRESULT => Some(EventTx { + tx: event.tx, + tx_type: PropagateTx, + }), } } } -//RcvTx processing -pub struct RcvTx(pub EventTx); -impl RcvTx { +//Processing of event that arrive: SourceTxType. +impl EventTx { pub async fn process_event( self, acl_whitelist: &impl AclWhitelist, - ) -> Result { - match self.0.validate_tx(acl_whitelist).await { - Ok(()) => Ok(DownloadTx(self.0)), + ) -> Result, EventProcessError> { + match self.validate_tx(acl_whitelist).await { + Ok(()) => Ok(self.into()), Err(err) => Err(EventProcessError::ValidateError(format!( "Tx validation fail:{err}" ))), } } + + //Tx validation process. + async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { + self.tx.validate()?; + + // Secondly verify that author is whitelisted. + if !acl_whitelist.contains(&self.tx.author).await? { + return Err(TransactionError::Validation( + "Tx permission denied signer not authorized".to_string(), + )); + } + + Ok(()) + } } //Download Tx processing -pub struct DownloadTx(EventTx); -impl DownloadTx { +impl EventTx { pub async fn process_event( self, local_directory_path: &PathBuf, http_peer_list: Vec<(SocketAddr, Option)>, - ) -> Result<(NewTx, Option), EventProcessError> { - let tx_hash = self.0.content.hash; + ) -> Result<(EventTx, Option>), EventProcessError> { + let tx_hash = self.tx.hash; let http_client = reqwest::Client::new(); - let asset_file_list = self - .0 - .content - .payload - .get_asset_list(tx_hash) - .map_err(|err| { - EventProcessError::DownloadAssetError(format!( - "Asset file param conversion error:{err}" - )) - })?; + let asset_file_list = self.tx.payload.get_asset_list(tx_hash).map_err(|err| { + EventProcessError::DownloadAssetError(format!( + "Asset file param conversion error:{err}" + )) + })?; let futures: Vec<_> = asset_file_list .into_iter() @@ -198,30 +139,47 @@ impl DownloadTx { .map_err(|err| { EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) })?; - let newtx = NewTx(self.0.transition(ConsistentTxType)); - let propagate = self.0.propagate(); + let newtx: EventTx = self.clone().into(); + let propagate: Option> = self.into(); Ok((newtx, propagate)) } } //Propagate Tx processing -pub struct PropagateTx(EventTx); -impl PropagateTx { +impl EventTx { pub async fn process_event( self, - p2p_sender: &UnboundedSender, + p2p_sender: &UnboundedSender>, ) -> Result<(), EventProcessError> { - let tx: Transaction = self.0.content.into(); + let tx = Transaction { + author: self.tx.author, + hash: self.tx.hash, + payload: self.tx.payload, + nonce: self.tx.nonce, + signature: self.tx.signature, + //TODO should be updated after the p2p send with a notification + propagated: true, + executed: self.tx.executed, + state: TxValdiated, + }; tracing::info!("Tx validation propagate tx:{}", tx.hash.to_string()); p2p_sender.send(tx).map_err(|err| err.into()) } } -//Save new Tx processing -pub struct NewTx(EventTx); -impl NewTx { +impl EventTx { pub async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { - let tx: Transaction = self.0.content.into(); + let tx = Transaction { + author: self.tx.author, + hash: self.tx.hash, + payload: self.tx.payload, + nonce: self.tx.nonce, + signature: self.tx.signature, + //TODO should be updated after the p2p send with a notification + propagated: true, + executed: self.tx.executed, + state: TxValdiated, + }; tracing::info!("Tx validation save tx:{}", tx.hash.to_string()); mempool .add(tx) diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index be4fe7df..1aa2ecf7 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -1,5 +1,8 @@ -use crate::txvalidation::event::{EventTx, RcvTx, SourceTxType}; -use crate::types::Transaction; +use crate::txvalidation::event::{EventTx, RcvTx}; +use crate::types::{ + transaction::{TxCreate, TxReceive, TxValdiated}, + Transaction, +}; use crate::Mempool; use futures_util::Stream; use futures_util::TryFutureExt; @@ -26,9 +29,12 @@ pub enum EventProcessError { #[error("Fail to rcv Tx from the channel: {0}")] RcvChannelError(#[from] tokio::sync::oneshot::error::RecvError), #[error("Fail to send the Tx on the channel: {0}")] - SendChannelError(#[from] tokio::sync::mpsc::error::SendError<(RcvTx, Option)>), + SendChannelError( + #[from] + tokio::sync::mpsc::error::SendError<(Transaction, Option)>, + ), #[error("Fail to send the Tx on the channel: {0}")] - PropagateTxError(#[from] tokio::sync::mpsc::error::SendError), + PropagateTxError(#[from] tokio::sync::mpsc::error::SendError>), #[error("validation fail: {0}")] ValidateError(String), #[error("Tx asset fail to download because {0}")] @@ -49,47 +55,43 @@ pub struct TxResultSender; //use to send a tx to the event process. #[derive(Debug, Clone)] pub struct TxEventSender { - sender: UnboundedSender<(RcvTx, Option)>, + sender: UnboundedSender<(Transaction, Option)>, _marker: PhantomData, } //Manage send from the p2p source impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + pub fn build( + sender: UnboundedSender<(Transaction, Option)>, + ) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::P2P, - }; + pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { self.sender - .send((RcvTx(event), None)) + .send((tx.into_received(TxReceive::P2P), None)) .map_err(|err| err.into()) } } //Manage send from the RPC source impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + pub fn build( + sender: UnboundedSender<(Transaction, Option)>, + ) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { let (sender, rx) = oneshot::channel(); - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::RPC, - }; self.sender - .send((RcvTx(event), Some(sender))) + .send((tx.into_received(TxReceive::RPC), Some(sender))) .map_err(|err| EventProcessError::from(err))?; rx.await? } @@ -97,21 +99,19 @@ impl TxEventSender { //Manage send from the Tx result execution source impl TxEventSender { - pub fn build(sender: UnboundedSender<(RcvTx, Option)>) -> Self { + pub fn build( + sender: UnboundedSender<(Transaction, Option)>, + ) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { let (sender, rx) = oneshot::channel(); - let event = EventTx { - content: tx.into(), - tx_type: SourceTxType::TXRESULT, - }; self.sender - .send((RcvTx(event), Some(sender))) + .send((tx.into_received(TxReceive::TXRESULT), Some(sender))) .map_err(|err| EventProcessError::from(err))?; rx.await? } @@ -130,9 +130,9 @@ pub async fn start_event_loop( ) -> eyre::Result<( JoinHandle<()>, //channel use to send RcvTx event to the processing - UnboundedSender<(RcvTx, Option)>, + UnboundedSender<(Transaction, Option)>, //output stream use to propagate Tx. - impl Stream, + impl Stream>, )> { let local_directory_path = Arc::new(local_directory_path); //start http download manager @@ -140,15 +140,19 @@ pub async fn start_event_loop( download_manager::serve_files(bind_addr, http_download_port, local_directory_path.clone()) .await?; - let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel::<(RcvTx, Option)>(); + let (tx, mut rcv_tx_event_rx) = + mpsc::unbounded_channel::<(Transaction, Option)>(); - let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::(); + let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::>(); let p2p_stream = UnboundedReceiverStream::new(p2p_recv); let jh = tokio::spawn({ let local_directory_path = local_directory_path.clone(); async move { - while let Some((event, callback)) = rcv_tx_event_rx.recv().await { + while let Some((tx, callback)) = rcv_tx_event_rx.recv().await { + //create new event with the Tx + let event: EventTx = tx.into(); + //process RcvTx(EventTx) event let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index ec088db5..8b94e66d 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -8,13 +8,13 @@ use std::path::PathBuf; //describe file data for a Tx #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] -pub struct TxFile { +pub struct DbFile { pub name: String, pub url: String, pub checksum: Hash, } -impl TxFile { +impl DbFile { pub fn try_from_prg_data( value: &transaction::ProgramData, ) -> Result, &'static str> { @@ -23,9 +23,9 @@ impl TxFile { file_name, file_url, checksum, - } => Some(TxFile { - url: file_url.clone(), + } => Some(DbFile { name: file_name.clone(), + url: file_url.clone(), checksum: checksum.clone().into(), }), transaction::ProgramData::Output { @@ -39,65 +39,146 @@ impl TxFile { Ok(file) } - - pub fn to_asset_file(self, tx_hash: Hash) -> AssetFile { - AssetFile::from_txfile(self, tx_hash) - } } -pub fn vec_txfile_to_bytes(vec: &[TxFile]) -> Result, bincode::Error> { - bincode::serialize(vec) -} - -//describe file data to calculate the file path for read/write -#[derive(Clone, Debug)] -pub struct AssetFile { - //true if the file is an image. - pub image: bool, - pub tx: Hash, +//describe file data for a Tx +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct File { pub name: String, pub url: String, pub checksum: Hash, + pub extention: E, } -impl AssetFile { - fn from_txfile(file: TxFile, tx_hash: Hash) -> Self { - AssetFile { - image: false, - url: file.url, - name: file.name, - tx: tx_hash, - checksum: file.checksum, +impl File { + pub fn build(name: String, url: String, checksum: Hash, extention: E) -> Self { + File { + name, + url, + checksum, + extention, } } +} + +//File type +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct Tx; + +impl File { + pub fn new(name: String, url: String, checksum: Hash) -> Self { + File::build(name, url, checksum, Tx) + } + + pub fn to_asset_file(self, tx_hash: Hash) -> File { + File::::from_txfile(self, tx_hash) + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct Download(Hash); +//Asset file use download and install the file data locally. +impl File { + pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { + File::build(name, url, checksum, Download(tx_hash)) + } + + fn from_txfile(file: File, tx_hash: Hash) -> Self { + File::::new(file.name, file.url, file.checksum, tx_hash) + } pub fn get_relatif_path(&self) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap(); - let mut path = if self.image { - PathBuf::from("images").join(self.tx.to_string()) - } else { - PathBuf::from(self.tx.to_string()) - }; + let mut path = PathBuf::from(self.extention.0.to_string()); path.push(file_name); path } - pub fn try_from_prg_meta_data( - value: &transaction::ProgramMetadata, - ) -> Result { - Ok(AssetFile { - image: true, - url: value.image_file_url.clone(), - name: value.name.clone(), - tx: value.hash, - checksum: value.image_file_checksum.clone().into(), - }) + pub fn try_from_prg_meta_data(value: &transaction::ProgramMetadata) -> Self { + File::::new( + value.name.clone(), + value.image_file_url.clone(), + value.image_file_checksum.clone().into(), + value.hash, + ) + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct ProofVerif; +impl File { + pub fn new(path: String, http_download_host: String, checksum: Hash) -> Self { + let uuid = uuid::Uuid::new_v4(); + //format file_name to keep the current filename and the uuid. + //put uuid at the end so that the uuid is use to save the file. + let new_file_name = format!("{}/{}", path, uuid); + + File::build(new_file_name, http_download_host, checksum, ProofVerif) + } + + pub fn into_download_file(self, tx_hash: Hash) -> File { + //get uuid from file name + let uuid = Path::new(&self.name).file_name().unwrap(); + File::::new( + self.name.to_string(), + format!("{}/{}/{}", self.url, tx_hash, uuid.to_str().unwrap()), + self.checksum.to_string().into(), + tx_hash, + ) + } + + pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { + let mut file_path = Path::new(&self.name); + if file_path.is_absolute() { + file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute + } + + let mut path = PathBuf::from(tx_hash.to_string()); + path.push(file_path); + path + } + + pub fn vec_to_bytes(vec: &[File]) -> Result, bincode::Error> { + bincode::serialize(vec) + } +} + +impl From for File { + fn from(file: DbFile) -> Self { + File::::new(file.name, file.url, file.checksum) + } +} + +//describe a file generated by the VM +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct Vm(Hash); +impl File { + pub fn new(path: String, checksum: Hash, task_tx: Hash) -> Self { + File::build(path, String::new(), checksum, Vm(task_tx)) + } + + pub fn get_relatif_path(&self) -> PathBuf { + let mut file_path = Path::new(&self.name); + if file_path.is_absolute() { + file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute + } + + let mut path = PathBuf::from(self.extention.0.to_string()); + path.push(file_path); + path } } -pub async fn move_vmfile(source: &VmFile, dest: &AssetFile, base_path: &Path) -> Result<()> { - let src_file_path = base_path.to_path_buf().join(&source.get_vmrelatif_path()); - let dst_file_path = base_path.to_path_buf().join(&dest.get_relatif_path()); +pub async fn move_vmfile( + source: &File, + dest: &File, + base_path: &Path, + proofverif_tx_hash: Hash, +) -> Result<()> { + let src_file_path = base_path.to_path_buf().join(source.get_relatif_path()); + let dst_file_path = base_path + .to_path_buf() + .join(dest.get_relatif_path(proofverif_tx_hash)); tracing::debug!( "moving file from {:#?} to {:#?}", @@ -117,22 +198,18 @@ pub async fn move_vmfile(source: &VmFile, dest: &AssetFile, base_path: &Path) -> .map_err(|e| e.into()) } -//describe file data to calculate the file path for read/write -#[derive(Clone, Debug)] -pub struct VmFile { - pub task_id: Hash, - pub name: String, -} - -impl VmFile { - pub fn get_vmrelatif_path(&self) -> PathBuf { - let mut file_path = Path::new(&self.name); - if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute - } +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] +pub struct Image(Hash); +impl File { + pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { + File::build(name, url, checksum, Image(tx_hash)) + } - let mut path = PathBuf::from(self.task_id.to_string()); - path.push(file_path); + pub fn get_relatif_path(&self) -> PathBuf { + let file_name = Path::new(&self.name).file_name().unwrap(); + let mut path = PathBuf::from("images"); + path.push(self.extention.0.to_string()); + path.push(file_name); path } } diff --git a/crates/node/src/types/task.rs b/crates/node/src/types/task.rs index 2ed37ad8..d1a96ec8 100644 --- a/crates/node/src/types/task.rs +++ b/crates/node/src/types/task.rs @@ -1,4 +1,4 @@ -use crate::types::file::TxFile; +use crate::types::file::DbFile; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -37,7 +37,7 @@ pub struct Task { pub program_id: Hash, pub args: Vec, #[sqlx(skip)] - pub files: Vec, + pub files: Vec, #[serde(skip_deserializing)] pub serial: i32, #[serde(skip_deserializing)] diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index f3fecb40..34aa2684 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -1,8 +1,6 @@ -use super::file::TxFile; +use super::file::{Download, File, ProofVerif}; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; -use crate::types::file::vec_txfile_to_bytes; -use crate::types::file::AssetFile; use crate::types::transaction; use async_trait::async_trait; use eyre::Result; @@ -10,7 +8,6 @@ use libsecp256k1::{sign, verify, Message, PublicKey, SecretKey}; use num_bigint::BigInt; use serde::{Deserialize, Serialize}; use sha3::{Digest, Sha3_256}; -use std::path::Path; use std::{collections::HashSet, rc::Rc}; use thiserror::Error; @@ -174,7 +171,7 @@ pub enum Payload { parent: Hash, prover: Hash, proof: Vec, - files: Vec, + files: Vec>, }, ProofKey { parent: Hash, @@ -184,7 +181,7 @@ pub enum Payload { parent: Hash, verifier: Hash, verification: Vec, - files: Vec, + files: Vec>, }, Cancel { parent: Hash, @@ -192,19 +189,13 @@ pub enum Payload { } impl Payload { - pub fn get_asset_list(&self, tx_hash: Hash) -> Result> { + pub fn get_asset_list(&self, tx_hash: Hash) -> Result>> { match self { transaction::Payload::Deploy { prover, verifier, .. } => Ok(vec![ - AssetFile::try_from_prg_meta_data(prover).map_err(|err| { - TransactionError::Validation(format!("Fail to get prover image file: {err}",)) - })?, - AssetFile::try_from_prg_meta_data(verifier).map_err(|err| { - TransactionError::Validation( - format!("Fail to gfet verifier image file: {err}",), - ) - })?, + File::::try_from_prg_meta_data(prover), + File::::try_from_prg_meta_data(verifier), ]), Payload::Run { workflow } => { workflow @@ -227,36 +218,19 @@ impl Payload { }) .map(|(file_name, file_url, checksum)| { //verify the url is valide. - reqwest::Url::parse(&file_url)?; - Ok(AssetFile { - image: false, - url: file_url.clone(), - name: file_name.to_string(), - tx: tx_hash, - checksum: checksum.to_string().into(), - }) + reqwest::Url::parse(file_url)?; + Ok(File::::new( + file_name.to_string(), + file_url.clone(), + checksum.to_string().into(), + tx_hash, + )) }) .collect() } Payload::Proof { files, .. } | Payload::Verification { files, .. } => files .iter() - .map(|file| { - //get uuid from file name - let uuid = Path::new(&file.name).file_name().unwrap(); - - Ok(AssetFile { - image: false, - url: format!( - "{}/{}/{}", - file.url, - tx_hash.to_string(), - uuid.to_str().unwrap() - ), - name: file.name.to_string(), - tx: tx_hash, - checksum: file.checksum.to_string().into(), - }) - }) + .map(|file| Ok(file.clone().into_download_file(tx_hash))) .collect(), // Other transaction types don't have external assets that would // need processing. @@ -299,7 +273,7 @@ impl Payload { buf.append(&mut prover.to_vec()); buf.append(proof.clone().as_mut()); buf.append(proof.clone().as_mut()); - buf.append(&mut vec_txfile_to_bytes(files).unwrap()); + buf.append(&mut File::::vec_to_bytes(files).unwrap()); } Payload::ProofKey { parent, key } => { buf.append(&mut parent.to_vec()); @@ -314,7 +288,7 @@ impl Payload { buf.append(&mut parent.to_vec()); buf.append(&mut verifier.to_vec()); buf.append(verification.clone().as_mut()); - buf.append(&mut vec_txfile_to_bytes(files).unwrap()); + buf.append(&mut File::::vec_to_bytes(files).unwrap()); } Payload::Cancel { parent } => { buf.append(&mut parent.to_vec()); @@ -333,7 +307,20 @@ pub enum TransactionError { } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct Transaction { +pub struct TxCreate; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub enum TxReceive { + P2P, + RPC, + TXRESULT, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct TxValdiated; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub struct Transaction { pub author: PublicKey, pub hash: Hash, pub payload: Payload, @@ -343,9 +330,22 @@ pub struct Transaction { pub propagated: bool, #[serde(skip_serializing, skip_deserializing)] pub executed: bool, + pub state: T, +} + +impl Transaction { + pub fn compute_hash(&self) -> Hash { + let mut hasher = Sha3_256::new(); + let mut buf = vec![]; + hasher.update(self.author.serialize()); + self.payload.serialize_into(&mut buf); + hasher.update(buf); + hasher.update(self.nonce.to_be_bytes()); + (&hasher.finalize()[0..32]).into() + } } -impl Default for Transaction { +impl Default for Transaction { fn default() -> Self { Self { author: PublicKey::from_secret_key(&SecretKey::default()), @@ -355,11 +355,27 @@ impl Default for Transaction { signature: Signature::default(), propagated: false, executed: false, + state: TxCreate, } } } -impl Transaction { +impl Default for Transaction { + fn default() -> Self { + Self { + author: PublicKey::from_secret_key(&SecretKey::default()), + hash: Hash::default(), + payload: Payload::default(), + nonce: 0, + signature: Signature::default(), + propagated: false, + executed: false, + state: TxValdiated, + } + } +} + +impl Transaction { pub fn new(payload: Payload, signing_key: &SecretKey) -> Self { let author = PublicKey::from_secret_key(signing_key); @@ -371,6 +387,7 @@ impl Transaction { signature: Signature::default(), propagated: false, executed: false, + state: TxCreate, }; tx.sign(signing_key); @@ -386,22 +403,27 @@ impl Transaction { self.signature = sig.into(); } + pub fn into_received(self, state: TxReceive) -> Transaction { + Transaction { + author: self.author, + hash: self.hash, + payload: self.payload, + nonce: self.nonce, + signature: self.signature, + propagated: self.propagated, + executed: self.executed, + state, + } + } +} + +impl Transaction { pub fn verify(&self) -> bool { let hash = self.compute_hash(); let msg: Message = hash.into(); verify(&msg, &self.signature.into(), &self.author) } - pub fn compute_hash(&self) -> Hash { - let mut hasher = Sha3_256::new(); - let mut buf = vec![]; - hasher.update(self.author.serialize()); - self.payload.serialize_into(&mut buf); - hasher.update(buf); - hasher.update(self.nonce.to_be_bytes()); - (&hasher.finalize()[0..32]).into() - } - pub fn validate(&self) -> Result<(), TransactionError> { if let Payload::Run { ref workflow } = self.payload { let mut programs = HashSet::new(); @@ -410,8 +432,7 @@ impl Transaction { return Err(TransactionError::Validation(format!( "multiple programs in workflow: {}", &step.program - )) - .into()); + ))); } } } @@ -419,8 +440,7 @@ impl Transaction { if !self.verify() { return Err(TransactionError::Validation(String::from( "signature verification failed", - )) - .into()); + ))); } Ok(()) @@ -439,6 +459,7 @@ mod tests { let sk = SecretKey::random(&mut StdRng::from_entropy()); let tx = Transaction::new(Payload::Empty, &sk); + let tx = tx.into_received(TxReceive::P2P); assert!(tx.verify()); } @@ -452,6 +473,7 @@ mod tests { tx.nonce += 1; // Verify must return false. + let tx = tx.into_received(TxReceive::TXRESULT); assert!(!tx.verify()); } @@ -466,31 +488,20 @@ mod tests { }; let sk = SecretKey::random(&mut StdRng::from_entropy()); - let tx = Transaction { - author: PublicKey::from_secret_key(&sk), - hash: Hash::default(), - payload: Payload::Run { workflow }, - nonce: 0, - signature: Signature::default(), - propagated: false, - executed: false, - }; + let mut tx = Transaction::::default(); + tx.author = PublicKey::from_secret_key(&sk); + tx.payload = Payload::Run { workflow }; + tx.signature = Signature::default(); + let tx = tx.into_received(TxReceive::RPC); assert!(tx.validate().is_err()); } #[test] fn test_tx_validations_verifies_signature() { - let tx = Transaction { - author: PublicKey::from_secret_key(&SecretKey::default()), - hash: Hash::default(), - payload: Payload::Empty, - nonce: 0, - signature: Signature::default(), - propagated: false, - executed: false, - }; + let tx = Transaction::::default(); + let tx = tx.into_received(TxReceive::RPC); assert!(tx.validate().is_err()); } } diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index 9f82c253..1bdd48d4 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -1,8 +1,8 @@ -use crate::types::file::TxFile; +use crate::types::file::DbFile; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::{ - transaction::{Payload, Workflow, WorkflowStep}, + transaction::{Payload, TxValdiated, Workflow, WorkflowStep}, Hash, Task, TaskKind, Transaction, }; use std::sync::Arc; @@ -30,7 +30,7 @@ pub enum WorkflowError { #[async_trait] pub trait TransactionStore: Sync + Send { - async fn find_transaction(&self, tx_hash: &Hash) -> Result>; + async fn find_transaction(&self, tx_hash: &Hash) -> Result>>; } pub struct WorkflowEngine { @@ -46,7 +46,7 @@ impl WorkflowEngine { } } - pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { + pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { let workflow = self.workflow_for_transaction(&cur_tx.hash).await?; match &cur_tx.payload { @@ -275,7 +275,7 @@ impl WorkflowEngine { .inputs .iter() .filter_map(|e| { - TxFile::try_from_prg_data(e) + DbFile::try_from_prg_data(e) .map_err(|err| WorkflowError::FileDefinitionError(err.to_string()).into()) .transpose() }) diff --git a/crates/tests/e2e-tests/src/main.rs b/crates/tests/e2e-tests/src/main.rs index 22b77de3..5f7f2fd8 100644 --- a/crates/tests/e2e-tests/src/main.rs +++ b/crates/tests/e2e-tests/src/main.rs @@ -1,3 +1,4 @@ +use gevulot_node::types::transaction::TxCreate; use std::{ net::SocketAddr, path::{Path, PathBuf}, @@ -99,7 +100,19 @@ async fn deploy_programs( .expect("get_transaction"); assert!(read_tx.is_some()); - assert_eq!(tx, read_tx.unwrap()); + let read_tx = read_tx.unwrap(); + let read_tx = Transaction { + author: read_tx.author, + hash: read_tx.hash, + payload: read_tx.payload, + nonce: read_tx.nonce, + signature: read_tx.signature, + propagated: false, + executed: false, + state: TxCreate, + }; + + assert_eq!(tx, read_tx); Ok((tx.hash, prover.hash, verifier.hash)) } From 4fc59b2435a4b5156215f2e7d80e5fb03bcba99e Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 12:40:35 +0100 Subject: [PATCH 08/43] correct some issues in file path --- crates/cli/src/lib.rs | 4 +- crates/node/src/scheduler/mod.rs | 5 +- crates/node/src/txvalidation/event.rs | 2 +- crates/node/src/types/file.rs | 46 +++++----- crates/node/src/types/transaction.rs | 117 +++++++++++++++----------- crates/node/src/vmm/qemu.rs | 1 + crates/node/src/workflow/mod.rs | 36 -------- 7 files changed, 98 insertions(+), 113 deletions(-) diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 798f36dd..474cc05e 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,5 +1,5 @@ use crate::file::FileData; -use gevulot_node::types::transaction::ProgramMetadata; +use gevulot_node::types::transaction::{ProgramMetadata, TxCreate}; use gevulot_node::types::Hash; use gevulot_node::{ rpc_client::RpcClient, @@ -203,7 +203,7 @@ pub async fn run_deploy_command( )) } -async fn send_transaction(client: &RpcClient, tx: &Transaction) -> Result { +async fn send_transaction(client: &RpcClient, tx: &Transaction) -> Result { client .send_transaction(tx) .await diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index b6534b62..c016a9ae 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -492,7 +492,7 @@ impl TaskManager for Scheduler { .cloned() .collect(); let nonce = rand::thread_rng().next_u64(); - let mut tx = match running_task.task.kind { + let tx = match running_task.task.kind { TaskKind::Proof => Transaction::new( Payload::Proof { parent: running_task.task.tx, @@ -522,9 +522,6 @@ impl TaskManager for Scheduler { } }; - //TODO Add tx execution in the state need some Higher Kind type. - tx.executed = true; - tracing::info!("Submit result Tx created:{}", tx.hash.to_string()); //Move tx file from execution Tx path to new Tx path diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index 6277ecab..0979ebf3 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -114,7 +114,7 @@ impl EventTx { ) -> Result<(EventTx, Option>), EventProcessError> { let tx_hash = self.tx.hash; let http_client = reqwest::Client::new(); - let asset_file_list = self.tx.payload.get_asset_list(tx_hash).map_err(|err| { + let asset_file_list = self.tx.get_asset_list(tx_hash).map_err(|err| { EventProcessError::DownloadAssetError(format!( "Asset file param conversion error:{err}" )) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 8b94e66d..5ffb72cd 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -76,11 +76,11 @@ impl File { } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub struct Download(Hash); +pub struct Download(String); //Asset file use download and install the file data locally. impl File { pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { - File::build(name, url, checksum, Download(tx_hash)) + File::build(name, url, checksum, Download(tx_hash.to_string())) } fn from_txfile(file: File, tx_hash: Hash) -> Self { @@ -89,19 +89,10 @@ impl File { pub fn get_relatif_path(&self) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap(); - let mut path = PathBuf::from(self.extention.0.to_string()); + let mut path = PathBuf::from(&self.extention.0); path.push(file_name); path } - - pub fn try_from_prg_meta_data(value: &transaction::ProgramMetadata) -> Self { - File::::new( - value.name.clone(), - value.image_file_url.clone(), - value.image_file_checksum.clone().into(), - value.hash, - ) - } } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] @@ -201,15 +192,30 @@ pub async fn move_vmfile( #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct Image(Hash); impl File { - pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { - File::build(name, url, checksum, Image(tx_hash)) + // pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { + // File::build(name, url, checksum, Image(tx_hash)) + // } + + pub fn try_from_prg_meta_data(value: &transaction::ProgramMetadata) -> Self { + File::build( + value.name.clone(), + value.image_file_url.clone(), + value.image_file_checksum.clone().into(), + Image(value.hash), + ) } +} - pub fn get_relatif_path(&self) -> PathBuf { - let file_name = Path::new(&self.name).file_name().unwrap(); - let mut path = PathBuf::from("images"); - path.push(self.extention.0.to_string()); - path.push(file_name); - path +impl From> for File { + fn from(file: File) -> Self { + //image file has the image directory happened at the beginning. + let mut extention = PathBuf::from("images"); + extention.push(file.extention.0.to_string()); + File::build( + file.name, + file.url, + file.checksum, + Download(extention.to_str().unwrap().to_string()), + ) } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 34aa2684..deeae412 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -1,4 +1,5 @@ -use super::file::{Download, File, ProofVerif}; +use super::file::{Download, File, Image, ProofVerif}; +use super::hash::Hash; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; use crate::types::transaction; @@ -189,55 +190,6 @@ pub enum Payload { } impl Payload { - pub fn get_asset_list(&self, tx_hash: Hash) -> Result>> { - match self { - transaction::Payload::Deploy { - prover, verifier, .. - } => Ok(vec![ - File::::try_from_prg_meta_data(prover), - File::::try_from_prg_meta_data(verifier), - ]), - Payload::Run { workflow } => { - workflow - .steps - .iter() - .flat_map(|step| &step.inputs) - .filter_map(|input| { - match input { - ProgramData::Input { - file_name, - file_url, - checksum, - } => Some((file_name, file_url, checksum)), - ProgramData::Output { .. } => { - /* ProgramData::Output as input means it comes from another - program execution -> skip this branch. */ - None - } - } - }) - .map(|(file_name, file_url, checksum)| { - //verify the url is valide. - reqwest::Url::parse(file_url)?; - Ok(File::::new( - file_name.to_string(), - file_url.clone(), - checksum.to_string().into(), - tx_hash, - )) - }) - .collect() - } - Payload::Proof { files, .. } | Payload::Verification { files, .. } => files - .iter() - .map(|file| Ok(file.clone().into_download_file(tx_hash))) - .collect(), - // Other transaction types don't have external assets that would - // need processing. - _ => Ok(vec![]), - } - } - pub fn serialize_into(&self, buf: &mut Vec) { match self { Payload::Empty => {} @@ -316,6 +268,15 @@ pub enum TxReceive { TXRESULT, } +impl TxReceive { + fn is_from_tx_exec_result(&self) -> bool { + match self { + TxReceive::TXRESULT => true, + _ => false, + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct TxValdiated; @@ -445,6 +406,62 @@ impl Transaction { Ok(()) } + + pub fn get_asset_list(&self, tx_hash: Hash) -> Result>> { + match &self.payload { + transaction::Payload::Deploy { + prover, verifier, .. + } => Ok(vec![ + File::::try_from_prg_meta_data(prover).into(), + File::::try_from_prg_meta_data(verifier).into(), + ]), + Payload::Run { workflow } => { + workflow + .steps + .iter() + .flat_map(|step| &step.inputs) + .filter_map(|input| { + match input { + ProgramData::Input { + file_name, + file_url, + checksum, + } => Some((file_name, file_url, checksum)), + ProgramData::Output { .. } => { + /* ProgramData::Output as input means it comes from another + program execution -> skip this branch. */ + None + } + } + }) + .map(|(file_name, file_url, checksum)| { + //verify the url is valide. + reqwest::Url::parse(file_url)?; + Ok(File::::new( + file_name.to_string(), + file_url.clone(), + checksum.to_string().into(), + tx_hash, + )) + }) + .collect() + } + Payload::Proof { files, .. } | Payload::Verification { files, .. } => { + //generated file during execution has already been moved. No Download. + if self.state.is_from_tx_exec_result() { + Ok(vec![]) + } else { + files + .iter() + .map(|file| Ok(file.clone().into_download_file(tx_hash))) + .collect() + } + } + // Other transaction types don't have external assets that would + // need processing. + _ => Ok(vec![]), + } + } } #[cfg(test)] diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 157ba019..2f6d30e0 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -141,6 +141,7 @@ impl Provider for Qemu { // TODO: // - Builder to construct QEMU flags // - Handle GPUs + // - verify the file exist before lauching. Avoid to panic the node because Qemu break. let img_file = Path::new(&self.config.data_directory) .join(IMAGES_DIR) diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index 1bdd48d4..f4f1c49e 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -279,44 +279,8 @@ impl WorkflowEngine { .map_err(|err| WorkflowError::FileDefinitionError(err.to_string()).into()) .transpose() }) - // match e { - // ProgramData::Input { - // file_name, - // file_url, - // .. - // } => File { - // tx, - // name: file_name.clone(), - // url: file_url.clone(), - // }, - // ProgramData::Output { - // source_program, - // file_name, - // } => { - // // Make record of file that needs transfer from source tx to current tx's files. - // file_transfers.push((*source_program, file_name.clone())); - // File { - // tx, - // name: file_name.clone(), - // url: "".to_string(), - // } - // } - // }) .collect::>>()?; - // // Process file transfers from source programs. - // //TODO! move end task execution. - // for (source_program, file_name) in file_transfers { - // let source_tx = self - // .find_parent_tx_for_program(&tx, &source_program) - // .await - // .expect("output file dependency missing"); - - // self.file_storage - // .move_task_file(&source_tx.to_string(), &tx.to_string(), &file_name) - // .await?; - // } - Ok(Task { id, tx, From 072235e9c300235b3c79e47ea4771215ed46de02 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 13:10:41 +0100 Subject: [PATCH 09/43] add some logs --- crates/node/src/txvalidation/event.rs | 3 +-- crates/node/src/types/transaction.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index 0979ebf3..aab28ae3 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -112,9 +112,8 @@ impl EventTx { local_directory_path: &PathBuf, http_peer_list: Vec<(SocketAddr, Option)>, ) -> Result<(EventTx, Option>), EventProcessError> { - let tx_hash = self.tx.hash; let http_client = reqwest::Client::new(); - let asset_file_list = self.tx.get_asset_list(tx_hash).map_err(|err| { + let asset_file_list = self.tx.get_asset_list().map_err(|err| { EventProcessError::DownloadAssetError(format!( "Asset file param conversion error:{err}" )) diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index deeae412..765a0944 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -407,7 +407,7 @@ impl Transaction { Ok(()) } - pub fn get_asset_list(&self, tx_hash: Hash) -> Result>> { + pub fn get_asset_list(&self) -> Result>> { match &self.payload { transaction::Payload::Deploy { prover, verifier, .. @@ -441,19 +441,24 @@ impl Transaction { file_name.to_string(), file_url.clone(), checksum.to_string().into(), - tx_hash, + self.hash, )) }) .collect() } Payload::Proof { files, .. } | Payload::Verification { files, .. } => { //generated file during execution has already been moved. No Download. + tracing::trace!( + "Payload::Proof tx:{} is_from_tx_exec_result:{}", + self.hash.to_string(), + self.state.is_from_tx_exec_result() + ); if self.state.is_from_tx_exec_result() { Ok(vec![]) } else { files .iter() - .map(|file| Ok(file.clone().into_download_file(tx_hash))) + .map(|file| Ok(file.clone().into_download_file(self.hash))) .collect() } } From 6428170de4fb1e79f08fd6e61a6719cc105d0c90 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 13:47:47 +0100 Subject: [PATCH 10/43] update proof generated file path --- crates/node/src/scheduler/mod.rs | 1 + crates/node/src/types/file.rs | 9 +++------ crates/node/src/types/transaction.rs | 8 +++++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index c016a9ae..64db3ab8 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -522,6 +522,7 @@ impl TaskManager for Scheduler { } }; + tracing::trace!("Submit result exec files:{}", executed_files.len()); tracing::info!("Submit result Tx created:{}", tx.hash.to_string()); //Move tx file from execution Tx path to new Tx path diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 5ffb72cd..2bd1c835 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -119,13 +119,10 @@ impl File { } pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { - let mut file_path = Path::new(&self.name); - if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute - } - + //get uuid from file name + let uuid = Path::new(&self.name).file_name().unwrap(); let mut path = PathBuf::from(tx_hash.to_string()); - path.push(file_path); + path.push(uuid); path } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 765a0944..9750b97e 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -447,12 +447,14 @@ impl Transaction { .collect() } Payload::Proof { files, .. } | Payload::Verification { files, .. } => { - //generated file during execution has already been moved. No Download. tracing::trace!( - "Payload::Proof tx:{} is_from_tx_exec_result:{}", + "Payload::Proof tx:{} is_from_tx_exec_result:{} files.len:{}", self.hash.to_string(), - self.state.is_from_tx_exec_result() + self.state.is_from_tx_exec_result(), + files.len(), ); + + //generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { Ok(vec![]) } else { From 3dd91cbfb5d0405633dc4d375acd84f3a902cd5e Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 14:46:57 +0100 Subject: [PATCH 11/43] remove comments --- crates/node/src/types/transaction.rs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 9750b97e..aadb9a54 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -447,13 +447,6 @@ impl Transaction { .collect() } Payload::Proof { files, .. } | Payload::Verification { files, .. } => { - tracing::trace!( - "Payload::Proof tx:{} is_from_tx_exec_result:{} files.len:{}", - self.hash.to_string(), - self.state.is_from_tx_exec_result(), - files.len(), - ); - //generated file during execution has already been moved. No Download. if self.state.is_from_tx_exec_result() { Ok(vec![]) From 2a714a13a3aaf7e31768684812fb9b4df5c24ba4 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 15:51:27 +0100 Subject: [PATCH 12/43] add comment, add timeout and error management to http download connection --- .../node/src/txvalidation/download_manager.rs | 32 +++++++++++++++++-- crates/node/src/types/file.rs | 26 ++++----------- crates/node/src/types/transaction.rs | 11 +++++++ 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 41afaad5..71465cdd 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -61,6 +61,7 @@ pub async fn download_asset_file( }; if resp.status() == reqwest::StatusCode::OK { + tracing::trace!("download_file:{} started.", asset_file.name); let file_path = local_directory_path.join(&local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { @@ -80,13 +81,38 @@ pub async fn download_asset_file( //create the Hasher to verify the Hash let mut hasher = blake3::Hasher::new(); - while let Some(chunk) = resp.chunk().await? { - hasher.update(&chunk); - fd.write_all(&chunk).await?; + loop { + match tokio::time::timeout(tokio::time::Duration::from_secs(5), resp.chunk()).await { + Ok(Ok(Some(chunk))) => { + hasher.update(&chunk); + fd.write_all(&chunk).await?; + } + Ok(Ok(None)) => break, + Ok(Err(_)) => { + tracing::error!("download_file:{:?} connection timeout.", asset_file.name); + return Err(eyre!( + "Download file: {:?}, connection timeout", + asset_file.name + )); + } + Err(err) => { + tracing::error!("download_file:{:?} http error:{err}.", asset_file.name); + return Err(eyre!( + "Download file: {:?}, http error:{err}", + asset_file.name + )); + } + } } + // while let Some(chunk) = resp.chunk().await? { + // hasher.update(&chunk); + // fd.write_all(&chunk).await?; + // } + fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); + tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { Err(eyre!("Download file: {:?}, bad checksum", asset_file.name)) } else { diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 2bd1c835..7f7d1bf9 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -6,7 +6,8 @@ use serde::Serialize; use std::path::Path; use std::path::PathBuf; -//describe file data for a Tx +//describe file data that is stored in the database. +// to manipulate file on disk use the equivalent type state definition File #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] pub struct DbFile { pub name: String, @@ -41,7 +42,11 @@ impl DbFile { } } -//describe file data for a Tx +// Type state definition of a file to manage all the different case of file manipulation. +// Download: Use to download the file data and move them to the right place. Constructed using the other type. +// ProofVerif: A file attached to a proof or verify Tx: Only downloaded on distant host. Nothing done on the host when the Tx has been executed. +// Vm(Hash): File generated by the VM execution. Move from Vm to ProofVerif place. +// Image: File attached to a Deploy Tx. Identify an image that are stored in the image directory. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct File { pub name: String, @@ -62,19 +67,6 @@ impl File { } //File type -#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub struct Tx; - -impl File { - pub fn new(name: String, url: String, checksum: Hash) -> Self { - File::build(name, url, checksum, Tx) - } - - pub fn to_asset_file(self, tx_hash: Hash) -> File { - File::::from_txfile(self, tx_hash) - } -} - #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct Download(String); //Asset file use download and install the file data locally. @@ -83,10 +75,6 @@ impl File { File::build(name, url, checksum, Download(tx_hash.to_string())) } - fn from_txfile(file: File, tx_hash: Hash) -> Self { - File::::new(file.name, file.url, file.checksum, tx_hash) - } - pub fn get_relatif_path(&self) -> PathBuf { let file_name = Path::new(&self.name).file_name().unwrap(); let mut path = PathBuf::from(&self.extention.0); diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index aadb9a54..e0c20775 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -258,6 +258,17 @@ pub enum TransactionError { General(String), } +// Transaction definition. +// Type state are use to define the different state of a Tx. +// +// Tx are defined in 3 domains: Validation, Execution, Storage. +// Currently the same definition is used but different type should be defined (TODO). +// Only the validation type state are defined. +// TxCreate : identify a Tx that has just been created. +// TxReceive: Identify a Tx that has been received. Determine the received source. +// TxValdiated: Identify a Tx that has been validated. Pass all the validation process. +// The validation suppose the Tx has been propagated. Currently there's no notification during the propagation (TODO). + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct TxCreate; From fdf618fce3fd3d5c00396cd502051c5c01264f91 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 16:14:13 +0100 Subject: [PATCH 13/43] correct http url definition --- crates/node/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index ddae8149..d4e2f39a 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -224,7 +224,7 @@ async fn run(config: Arc) -> Result<()> { let workflow_engine = Arc::new(WorkflowEngine::new(database.clone())); let download_url_prefix = format!( "http://{}:{}", - config.p2p_listen_addr.to_string(), + config.p2p_listen_addr.ip().to_string(), config.http_download_port ); From ff9e291a8481e63271dea51f69b32ad4020bb89a Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 16:19:49 +0100 Subject: [PATCH 14/43] add timeout to http send --- .../node/src/txvalidation/download_manager.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 71465cdd..f468f056 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -29,9 +29,14 @@ pub async fn download_asset_file( ) -> Result<()> { let local_relative_file_path = asset_file.get_relatif_path(); tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); - let mut resp = match http_client.get(asset_file.url).send().await { - Ok(resp) => resp, - Err(err) => { + let mut resp = match tokio::time::timeout( + tokio::time::Duration::from_secs(5), + http_client.get(asset_file.url).send(), + ) + .await + { + Ok(Ok(resp)) => resp, + Ok(Err(err)) => { let peer_urls: Vec = http_peer_list .iter() .filter_map(|(peer, port)| { @@ -58,6 +63,13 @@ pub async fn download_asset_file( asset_file.name ))? } + Err(err) => { + tracing::error!("download_file:{:?} request send timeout.", asset_file.name); + return Err(eyre!( + "Download file: {:?}, request send timeout.", + asset_file.name + )); + } }; if resp.status() == reqwest::StatusCode::OK { From c58ad9da146863c0b34f0bd9c536230702eed022 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 16:40:13 +0100 Subject: [PATCH 15/43] add better download error login and log file checksum --- crates/node/src/scheduler/mod.rs | 1 + crates/node/src/txvalidation/download_manager.rs | 9 +++++---- crates/node/src/txvalidation/event.rs | 1 + 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 64db3ab8..0e6671de 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -527,6 +527,7 @@ impl TaskManager for Scheduler { //Move tx file from execution Tx path to new Tx path for (source_file, dest_file) in executed_files { + log::trace!("Move file {dest_file.name} checksum:{dest_file.checksum}"); if let Err(err) = move_vmfile(&source_file, &dest_file, &self.data_directory, tx.hash).await { diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index f468f056..e98e7ca2 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -64,7 +64,6 @@ pub async fn download_asset_file( ))? } Err(err) => { - tracing::error!("download_file:{:?} request send timeout.", asset_file.name); return Err(eyre!( "Download file: {:?}, request send timeout.", asset_file.name @@ -101,14 +100,12 @@ pub async fn download_asset_file( } Ok(Ok(None)) => break, Ok(Err(_)) => { - tracing::error!("download_file:{:?} connection timeout.", asset_file.name); return Err(eyre!( "Download file: {:?}, connection timeout", asset_file.name )); } Err(err) => { - tracing::error!("download_file:{:?} http error:{err}.", asset_file.name); return Err(eyre!( "Download file: {:?}, http error:{err}", asset_file.name @@ -126,7 +123,11 @@ pub async fn download_asset_file( let checksum: crate::types::Hash = (&hasher.finalize()).into(); tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { - Err(eyre!("Download file: {:?}, bad checksum", asset_file.name)) + Err(eyre!( + "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", + asset_file.name, + asset_file.checksum + )) } else { //rename to original name Ok(std::fs::rename(tmp_file_path, file_path)?) diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index aab28ae3..97621d5a 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -136,6 +136,7 @@ impl EventTx { .map(|res| res) .collect::, _>>() .map_err(|err| { + tracing::error!("Error during Tx file download:{err}"); EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) })?; let newtx: EventTx = self.clone().into(); From b19c14fce67c8011e464d326e42a63f9218ccb99 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 16:44:10 +0100 Subject: [PATCH 16/43] correct build --- crates/node/src/scheduler/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 0e6671de..be100b03 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -527,7 +527,11 @@ impl TaskManager for Scheduler { //Move tx file from execution Tx path to new Tx path for (source_file, dest_file) in executed_files { - log::trace!("Move file {dest_file.name} checksum:{dest_file.checksum}"); + tracing::trace!( + "Move file {} checksum:{}", + dest_file.name, + dest_file.checksum + ); if let Err(err) = move_vmfile(&source_file, &dest_file, &self.data_directory, tx.hash).await { From 423d5736f72f0a76b237714e11f13103f16be713 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 17:12:20 +0100 Subject: [PATCH 17/43] desactivate checksum verification for downloaded files --- crates/node/src/txvalidation/download_manager.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index e98e7ca2..7a68aec2 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -123,11 +123,13 @@ pub async fn download_asset_file( let checksum: crate::types::Hash = (&hasher.finalize()).into(); tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { - Err(eyre!( - "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", - asset_file.name, - asset_file.checksum - )) + //TODO desactivate checksum verification for now + Ok(std::fs::rename(tmp_file_path, file_path)?) + // Err(eyre!( + // "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", + // asset_file.name, + // asset_file.checksum + // )) } else { //rename to original name Ok(std::fs::rename(tmp_file_path, file_path)?) From 14bbd3c70c1a7db141d46e1c87f21cef407f7c2e Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 17:17:36 +0100 Subject: [PATCH 18/43] add some logs for checksum --- crates/node/src/txvalidation/download_manager.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 7a68aec2..c9f61cd1 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -120,7 +120,14 @@ pub async fn download_asset_file( // } fd.flush().await?; - let checksum: crate::types::Hash = (&hasher.finalize()).into(); + let vec_hash = hasher.finalize(); + tracing::trace!( + "download_file:{} Ended vec_hash:{:?}.", + asset_file.name, + vec_hash + ); + + let checksum: crate::types::Hash = (&vec_hash).into(); tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { //TODO desactivate checksum verification for now From 4557493a7f45827773a7bc85ddfe63359af41a83 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 18:52:12 +0100 Subject: [PATCH 19/43] change hasher for VM file and activate checksum verification --- Cargo.lock | 2 +- .../node/src/txvalidation/download_manager.rs | 26 +++++-------------- crates/shim/Cargo.toml | 2 +- crates/shim/src/lib.rs | 9 +++---- 4 files changed, 12 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ee856de..74d5c9f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1202,8 +1202,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-stream", + "blake3", "prost 0.11.9", - "sha3", "tokio", "tokio-stream", "tokio-vsock", diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index c9f61cd1..e3c4adba 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -114,29 +114,15 @@ pub async fn download_asset_file( } } - // while let Some(chunk) = resp.chunk().await? { - // hasher.update(&chunk); - // fd.write_all(&chunk).await?; - // } - fd.flush().await?; - let vec_hash = hasher.finalize(); - tracing::trace!( - "download_file:{} Ended vec_hash:{:?}.", - asset_file.name, - vec_hash - ); - - let checksum: crate::types::Hash = (&vec_hash).into(); + let checksum: crate::types::Hash = (&hasher.finalize()).into(); tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { - //TODO desactivate checksum verification for now - Ok(std::fs::rename(tmp_file_path, file_path)?) - // Err(eyre!( - // "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", - // asset_file.name, - // asset_file.checksum - // )) + Err(eyre!( + "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", + asset_file.name, + asset_file.checksum + )) } else { //rename to original name Ok(std::fs::rename(tmp_file_path, file_path)?) diff --git a/crates/shim/Cargo.toml b/crates/shim/Cargo.toml index a261c6d1..72388086 100644 --- a/crates/shim/Cargo.toml +++ b/crates/shim/Cargo.toml @@ -7,8 +7,8 @@ license = "MIT OR Apache-2.0" [dependencies] anyhow = "1" async-stream = "0.3.5" +blake3 = "1.5" prost = "0.11" -sha3 = "0.10" tokio = { version = "1.0", features = ["fs", "macros", "rt-multi-thread"] } tokio-stream = "0.1" tokio-vsock = { version = "0.4.0", features = ["tonic-conn"] } diff --git a/crates/shim/src/lib.rs b/crates/shim/src/lib.rs index a693d777..c56f2c53 100644 --- a/crates/shim/src/lib.rs +++ b/crates/shim/src/lib.rs @@ -1,6 +1,5 @@ use grpc::vm_service_client::VmServiceClient; use grpc::{FileChunk, FileData, FileMetadata}; -use sha3::{Digest, Sha3_256}; use std::fs::File; use std::io::{BufRead, BufReader}; use std::path::PathBuf; @@ -155,8 +154,8 @@ impl GRPCClient { Ok(Some(task)) } - fn submit_file(&mut self, task_id: TaskId, file_path: String) -> Result> { - let hasher = Arc::new(Mutex::new(Sha3_256::new())); + fn submit_file(&mut self, task_id: TaskId, file_path: String) -> Result<[u8; 32]> { + let hasher = Arc::new(Mutex::new(blake3::Hasher::new())); self.rt.block_on(async { let stream_hasher = Arc::clone(&hasher); let outbound = async_stream::stream! { @@ -200,7 +199,7 @@ impl GRPCClient { } }); let hasher = Arc::into_inner(hasher).unwrap().into_inner(); - let hash = hasher.finalize().to_vec(); + let hash = hasher.finalize().into(); Ok(hash) } @@ -213,7 +212,7 @@ impl GRPCClient { self.submit_file(result.id.clone(), file.clone()) .map(|checksum| crate::grpc::File { path: file.to_string(), - checksum, + checksum: checksum.to_vec(), }) }) .collect::>>()?; From 48aec375faefec48c74899e37d1fcd7c1fa96621 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 20:47:05 +0100 Subject: [PATCH 20/43] correct host detection for download --- crates/node/src/txvalidation/download_manager.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index e3c4adba..4ae8a20d 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -53,9 +53,11 @@ pub async fn download_asset_file( let mut resp = None; for url in peer_urls { if let Ok(val) = http_client.get(url.clone()).send().await { - tracing::trace!("download_file from peer url:{url}"); - resp = Some(val); - break; + if let reqwest::StatusCode::OK = val.status() { + tracing::trace!("download_file from peer url:{url}"); + resp = Some(val); + break; + } } } resp.ok_or(eyre!( From 32c5c0e5acb0fa766ef033d5316a6b108525e2e3 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 12 Feb 2024 21:33:53 +0100 Subject: [PATCH 21/43] disable proof and verifcation workflow and remove some logs --- crates/node/src/scheduler/mod.rs | 2 - .../node/src/txvalidation/download_manager.rs | 2 - crates/node/src/vmm/vm_server.rs | 1 - crates/node/src/workflow/mod.rs | 37 ++++++++++++------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index be100b03..e65995f3 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -521,8 +521,6 @@ impl TaskManager for Scheduler { ); } }; - - tracing::trace!("Submit result exec files:{}", executed_files.len()); tracing::info!("Submit result Tx created:{}", tx.hash.to_string()); //Move tx file from execution Tx path to new Tx path diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 4ae8a20d..33b3f545 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -74,7 +74,6 @@ pub async fn download_asset_file( }; if resp.status() == reqwest::StatusCode::OK { - tracing::trace!("download_file:{} started.", asset_file.name); let file_path = local_directory_path.join(&local_relative_file_path); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { @@ -118,7 +117,6 @@ pub async fn download_asset_file( fd.flush().await?; let checksum: crate::types::Hash = (&hasher.finalize()).into(); - tracing::trace!("download_file:{} Ended.", asset_file.name); if checksum != asset_file.checksum { Err(eyre!( "download_file:{:?} bad checksum checksum:{checksum} set_file.checksum:{}.", diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index bc3175a4..9e543b84 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -226,7 +226,6 @@ impl VmService for VMServer { .join(self.file_storage.data_dir()) .join(task_id) .join(path); - tracing::trace!("VM submit_file saved in {file_path:#?}"); // Ensure any necessary subdirectories exists. if let Some(parent) = file_path.parent() { diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index f4f1c49e..1c669e00 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -47,7 +47,11 @@ impl WorkflowEngine { } pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { - let workflow = self.workflow_for_transaction(&cur_tx.hash).await?; + let workflow = match self.workflow_for_transaction(&cur_tx.hash).await? { + Some(w) => w, + //no workflow assocaited return. + None => return Ok(None), + }; match &cur_tx.payload { Payload::Run { workflow } => { @@ -218,7 +222,7 @@ impl WorkflowEngine { } } - async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result { + async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result> { let mut tx_hash = *tx_hash; tracing::debug!("finding workflow for transaction {}", tx_hash); @@ -235,12 +239,14 @@ impl WorkflowEngine { match tx.unwrap().payload { Payload::Run { workflow } => { tracing::debug!("workflow found for transaction {}", tx_hash); - return Ok(workflow); + return Ok(Some(workflow)); } Payload::Proof { parent, .. } => { - tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); - tx_hash = parent; - continue; + return Ok(None); + // tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); + //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db + // tx_hash = parent; + // continue; } Payload::ProofKey { parent, .. } => { tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); @@ -248,16 +254,19 @@ impl WorkflowEngine { continue; } Payload::Verification { parent, .. } => { - tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); - tx_hash = parent; - continue; + return Ok(None); + //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db + // tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); + // tx_hash = parent; + // continue; } payload => { - tracing::debug!( - "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", - &tx_hash - ); - return Err(WorkflowError::IncompatibleTransaction(tx_hash.to_string()).into()); + return Ok(None); + // tracing::debug!( + // "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", + // &tx_hash + // ); + // return Err(WorkflowError::IncompatibleTransaction(tx_hash.to_string()).into()); } } } From 7bce8e6b21b2b308a57ac570a2aa5f5503648a8b Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 13 Feb 2024 09:18:54 +0100 Subject: [PATCH 22/43] rebase from master and correct download file issue --- crates/node/src/scheduler/program_manager.rs | 2 + .../node/src/txvalidation/download_manager.rs | 4 +- crates/node/src/types/transaction.rs | 1 - crates/node/src/workflow/mod.rs | 37 ++++++++----------- 4 files changed, 19 insertions(+), 25 deletions(-) diff --git a/crates/node/src/scheduler/program_manager.rs b/crates/node/src/scheduler/program_manager.rs index 96702937..2820d063 100644 --- a/crates/node/src/scheduler/program_manager.rs +++ b/crates/node/src/scheduler/program_manager.rs @@ -60,6 +60,8 @@ impl ProgramManager { id: Hash, limits: Option, ) -> Result { + tracing::trace!("start_program task:{}", id.to_string()); + let program = match self.storage.find_program(&id).await? { Some(program) => program, None => return Err(ProgramError::ProgramNotFound(id.to_string()).into()), diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 33b3f545..4ba64309 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -35,8 +35,8 @@ pub async fn download_asset_file( ) .await { - Ok(Ok(resp)) => resp, - Ok(Err(err)) => { + Ok(Ok(resp)) if resp.status() == reqwest::StatusCode::OK => resp, + Ok(_) => { let peer_urls: Vec = http_peer_list .iter() .filter_map(|(peer, port)| { diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index e0c20775..a6db2ee8 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -1,5 +1,4 @@ use super::file::{Download, File, Image, ProofVerif}; -use super::hash::Hash; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; use crate::types::transaction; diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index 1c669e00..fb6b1e02 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -47,11 +47,7 @@ impl WorkflowEngine { } pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { - let workflow = match self.workflow_for_transaction(&cur_tx.hash).await? { - Some(w) => w, - //no workflow assocaited return. - None => return Ok(None), - }; + let workflow = self.workflow_for_transaction(&cur_tx.hash).await?; match &cur_tx.payload { Payload::Run { workflow } => { @@ -80,7 +76,7 @@ impl WorkflowEngine { match workflow.steps.iter().position(|s| s.program == *prover) { Some(proof_step_idx) => { - if proof_step_idx <= workflow.steps.len() { + if workflow.steps.len() <= proof_step_idx { Err(WorkflowError::WorkflowStepMissing(format!( "verifier for proof tx {}", cur_tx.hash.clone(), @@ -222,7 +218,7 @@ impl WorkflowEngine { } } - async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result> { + async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result { let mut tx_hash = *tx_hash; tracing::debug!("finding workflow for transaction {}", tx_hash); @@ -239,14 +235,13 @@ impl WorkflowEngine { match tx.unwrap().payload { Payload::Run { workflow } => { tracing::debug!("workflow found for transaction {}", tx_hash); - return Ok(Some(workflow)); + return Ok(workflow); } Payload::Proof { parent, .. } => { - return Ok(None); - // tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db - // tx_hash = parent; - // continue; + tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); + tx_hash = parent; + continue; } Payload::ProofKey { parent, .. } => { tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); @@ -254,19 +249,17 @@ impl WorkflowEngine { continue; } Payload::Verification { parent, .. } => { - return Ok(None); //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db - // tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); - // tx_hash = parent; - // continue; + tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); + tx_hash = parent; + continue; } payload => { - return Ok(None); - // tracing::debug!( - // "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", - // &tx_hash - // ); - // return Err(WorkflowError::IncompatibleTransaction(tx_hash.to_string()).into()); + tracing::debug!( + "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", + &tx_hash + ); + return Err(WorkflowError::IncompatibleTransaction(tx_hash.to_string()).into()); } } } From f8be0dfd20a6bd77611c0cb6f925c8639680ca17 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 13 Feb 2024 09:34:31 +0100 Subject: [PATCH 23/43] rebase from master and correct download file issue --- crates/node/src/asset_manager/mod.rs | 190 --------------------------- crates/node/src/main.rs | 4 +- crates/node/src/storage/file.rs | 104 --------------- crates/node/src/storage/mod.rs | 3 - crates/node/src/types/file.rs | 14 ++ crates/node/src/vmm/vm_server.rs | 41 +++--- 6 files changed, 33 insertions(+), 323 deletions(-) delete mode 100644 crates/node/src/asset_manager/mod.rs delete mode 100644 crates/node/src/storage/file.rs diff --git a/crates/node/src/asset_manager/mod.rs b/crates/node/src/asset_manager/mod.rs deleted file mode 100644 index c2e2675c..00000000 --- a/crates/node/src/asset_manager/mod.rs +++ /dev/null @@ -1,190 +0,0 @@ -//TODO to be removed -use crate::{ - cli::Config, - storage::Database, - types::{transaction::Transaction, Hash}, -}; -use eyre::Result; -use gevulot_node::types::transaction::TxValdiated; -use std::collections::HashMap; -use std::net::SocketAddr; -use std::{sync::Arc, time::Duration}; -use thiserror::Error; -use tokio::time::sleep; - -#[allow(clippy::enum_variant_names)] -#[derive(Error, Debug)] -enum AssetManagerError { - #[error("program image download")] - ProgramImageDownload, - - #[error("incompatible transaction payload")] - IncompatibleTxPayload(Hash), -} - -/// AssetManager is reponsible for coordinating asset management for a new -/// transaction. New deployment of prover & verifier requires downloading of -/// VM images for those programs. Similarly, Run transaction has associated -/// input data which must be downloaded, but also built into workspace volume -/// for execution. -pub struct AssetManagerOld { - config: Arc, - database: Arc, - http_client: reqwest::Client, - http_peer_list: Arc>>>, -} - -impl AssetManagerOld { - pub fn new( - config: Arc, - database: Arc, - http_peer_list: Arc>>>, - ) -> Self { - Self { - config, - database, - http_client: reqwest::Client::new(), - http_peer_list, - } - } - - pub async fn run(&self) -> Result<()> { - // Main processing loop. - loop { - for tx_hash in self.database.get_incomplete_assets().await? { - if let Some(tx) = self.database.find_transaction(&tx_hash).await? { - // if let Err(err) = self.process_transaction(&tx).await { - // tracing::error!( - // "failed to process transaction (hash: {}) assets: {}", - // tx.hash, - // err - // ); - // continue; - // } - - self.database.mark_asset_complete(&tx_hash).await?; - } else { - tracing::warn!("asset entry for missing transaction; hash: {}", &tx_hash); - } - } - - // TODO: Define specific period for Asset processing refresh and - // compute remaining sleep time from that. If asset processing - // takes longer than anticipated, then there's no sleep. Otherwise - // next iteration starts from same periodic cycle as normally. - sleep(Duration::from_millis(500)).await; - } - } - - /// handle_transaction admits transaction into `AssetManager` for further - /// processing. - pub async fn handle_transaction(&self, tx: &Transaction) -> Result<()> { - self.database.add_asset(&tx.hash).await - } - - // async fn process_transaction(&self, tx: &Transaction) -> Result<()> { - // match tx.payload { - // transaction::Payload::Deploy { .. } => self.process_deployment(tx).await, - // transaction::Payload::Run { .. } => self.process_run(tx).await, - // // Other transaction types don't have external assets that would - // // need processing. - // _ => Ok(()), - // } - // } - - // async fn process_deployment(&self, tx: &Transaction) -> Result<()> { - // let (prover, verifier) = match tx.payload.clone() { - // Payload::Deploy { - // name: _, - // prover, - // verifier, - // } => (Program::from(prover), Program::from(verifier)), - // _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), - // }; - - // self.process_program(&prover).await?; - // self.process_program(&verifier).await?; - - // Ok(()) - // } - - // async fn process_program(&self, program: &Program) -> Result<()> { - // self.download_image(program).await - // } - - // async fn process_run(&self, tx: &Transaction) -> Result<()> { - // let workflow = match tx.payload.clone() { - // Payload::Run { workflow } => workflow, - // _ => return Err(AssetManagerError::IncompatibleTxPayload(tx.hash).into()), - // }; - - // // TODO: Ideally the following would happen concurrently for each file... - // for step in workflow.steps { - // for input in step.inputs { - // match input { - // ProgramData::Input { - // file_name, - // file_url, - // checksum, - // } => { - // let f = types::File { - // tx: tx.hash, - // name: file_name, - // url: file_url, - // }; - // crate::networking::download_manager::download_file( - // &f.url, - // &self.config.data_directory, - // f.get_file_relative_path() - // .to_str() - // .ok_or(eyre!("Download bad file path: {:?}", f.name))?, - // self.get_peer_list().await, - // &self.http_client, - // checksum.into(), - // ) - // .await?; - // } - // ProgramData::Output { .. } => { - // /* ProgramData::Output asinput means it comes from another - // program execution -> skip this branch. */ - // } - // } - // } - // } - - // Ok(()) - // } - - // /// download downloads file from the given `url` and saves it to file in `file_path`. - // async fn download_image(&self, program: &Program) -> Result<()> { - // let file_path = PathBuf::new() - // .join("images") - // .join(program.hash.to_string()) - // .join(&program.image_file_name); - // tracing::info!( - // "asset download url:{} file_path:{file_path:?} file_checksum:{}", - // program.image_file_url, - // program.image_file_checksum - // ); - // crate::networking::download_manager::download_file( - // &program.image_file_url, - // &self.config.data_directory, - // file_path - // .to_str() - // .ok_or(eyre!("Download bad file path: {:?}", file_path))?, - // self.get_peer_list().await, - // &self.http_client, - // (&*program.image_file_checksum).into(), - // ) - // .await - // } - - // async fn get_peer_list(&self) -> Vec<(SocketAddr, Option)> { - // self.http_peer_list - // .read() - // .await - // .iter() - // .map(|(a, p)| (*a, *p)) - // .collect() - // } -} diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index d4e2f39a..122ef9b9 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -31,7 +31,6 @@ use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan, EnvFilter}; use types::{transaction::TxValdiated, Hash, Transaction}; use workflow::WorkflowEngine; -mod asset_manager; mod cli; mod mempool; mod nanos; @@ -241,9 +240,8 @@ async fn run(config: Arc) -> Result<()> { txvalidation::TxEventSender::::build(tx_sender.clone()), )); - let file_storage = Arc::new(storage::File::new(&config.data_directory)); let vm_server = - vmm::vm_server::VMServer::new(scheduler.clone(), provider, file_storage.clone()); + vmm::vm_server::VMServer::new(scheduler.clone(), provider, config.data_directory.clone()); // Start gRPC VSOCK server. tokio::spawn(async move { diff --git a/crates/node/src/storage/file.rs b/crates/node/src/storage/file.rs deleted file mode 100644 index fa2ba854..00000000 --- a/crates/node/src/storage/file.rs +++ /dev/null @@ -1,104 +0,0 @@ -use eyre::Result; -use std::path::{Path, PathBuf}; -use tokio::io::AsyncWriteExt; - -pub struct File { - data_dir: PathBuf, -} - -impl File { - pub fn new(data_dir: &Path) -> Self { - File { - data_dir: PathBuf::new().join(data_dir), - } - } - - pub fn data_dir(&self) -> PathBuf { - self.data_dir.clone() - } - - pub async fn get_task_file( - &self, - task_id: &str, - path: &str, - ) -> Result> { - let mut path = Path::new(path); - if path.is_absolute() { - path = path.strip_prefix("/")?; - } - let path = PathBuf::new().join(&self.data_dir).join(task_id).join(path); - let fd = tokio::fs::File::open(path).await?; - Ok(tokio::io::BufReader::new(fd)) - } - - pub async fn move_task_file( - &self, - task_id_src: &str, - task_id_dst: &str, - path: &str, - ) -> Result<()> { - let mut path = Path::new(path); - if path.is_absolute() { - path = path.strip_prefix("/")?; - } - - let src_file_path = PathBuf::new() - .join(&self.data_dir) - .join(task_id_src) - .join(path); - let dst_file_path = PathBuf::new() - .join(&self.data_dir) - .join(task_id_dst) - .join(path); - - tracing::debug!( - "moving file from {:#?} to {:#?}", - src_file_path, - dst_file_path - ); - - // Ensure any necessary subdirectories exists. - if let Some(parent) = dst_file_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .expect("task file mkdir"); - } - - tokio::fs::rename(src_file_path, dst_file_path) - .await - .map_err(|e| e.into()) - } - - pub async fn save_task_file(&self, task_id: &str, path: &str, data: Vec) -> Result<()> { - let mut path = Path::new(path); - if path.is_absolute() { - path = path.strip_prefix("/")?; - } - - let file_path = PathBuf::new().join(&self.data_dir).join(task_id).join(path); - - tracing::debug!( - "saving task {} file {:#?} to {:#?}", - task_id, - path, - file_path - ); - - // Ensure any necessary subdirectories exists. - if let Some(parent) = file_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .expect("task file mkdir"); - } - - let fd = tokio::fs::File::create(&file_path).await?; - let mut fd = tokio::io::BufWriter::new(fd); - - fd.write_all(data.as_slice()).await?; - fd.flush().await?; - - tracing::debug!("file {:#?} successfully written", file_path); - - Ok(()) - } -} diff --git a/crates/node/src/storage/mod.rs b/crates/node/src/storage/mod.rs index af53df79..ff48cd81 100644 --- a/crates/node/src/storage/mod.rs +++ b/crates/node/src/storage/mod.rs @@ -1,5 +1,2 @@ pub(crate) mod database; -mod file; - pub use database::postgres::Database; -pub use file::File; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 7f7d1bf9..39bad833 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -6,6 +6,20 @@ use serde::Serialize; use std::path::Path; use std::path::PathBuf; +pub async fn open_task_file( + data_dir: &PathBuf, + task_id: &str, + path: &str, +) -> Result> { + let mut path = Path::new(path); + if path.is_absolute() { + path = path.strip_prefix("/")?; + } + let path = PathBuf::new().join(data_dir).join(task_id).join(path); + let fd = tokio::fs::File::open(path).await?; + Ok(tokio::io::BufReader::new(fd)) +} + //describe file data that is stored in the database. // to manipulate file on disk use the equivalent type state definition File #[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize, sqlx::FromRow)] diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index 9e543b84..b289c6ad 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -1,21 +1,19 @@ +use crate::types::file; +use crate::types::Hash; +use crate::vmm::vm_server::grpc::file_data; +use async_trait::async_trait; +use eyre::Result; +use grpc::vm_service_server::VmService; +use grpc::{GetFileRequest, Task, TaskRequest, TaskResultResponse}; use std::fmt::Debug; use std::path::{Path, PathBuf}; use std::sync::Arc; - -use async_trait::async_trait; -use eyre::Result; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; use tokio::sync::{mpsc, Mutex}; use tokio_stream::wrappers::ReceiverStream; use tonic::{Code, Extensions, Request, Response, Status, Streaming}; -use grpc::vm_service_server::VmService; -use grpc::{GetFileRequest, Task, TaskRequest, TaskResultResponse}; - -use crate::storage; -use crate::types::Hash; -use crate::vmm::vm_server::grpc::file_data; - use self::grpc::{ FileChunk, FileData, FileMetadata, GenericResponse, TaskResponse, TaskResultRequest, }; @@ -55,7 +53,7 @@ pub trait ProgramRegistry: Send { pub struct VMServer { task_source: Arc, program_registry: Arc>, - file_storage: Arc, + file_data_dir: PathBuf, } impl Debug for VMServer { @@ -68,12 +66,12 @@ impl VMServer { pub fn new( task_source: Arc, program_registry: Arc>, - file_storage: Arc, + file_data_dir: PathBuf, ) -> Self { VMServer { task_source, program_registry, - file_storage, + file_data_dir, } } @@ -138,14 +136,11 @@ impl VmService for VMServer { let req = request.into_inner(); - let mut file = match self - .file_storage - .get_task_file(&req.task_id, &req.path) - .await - { - Ok(file) => file, - Err(err) => return Err(Status::new(Code::NotFound, "couldn't get task file")), - }; + let mut file = + match file::open_task_file(&self.file_data_dir, &req.task_id, &req.path).await { + Ok(file) => file, + Err(err) => return Err(Status::new(Code::NotFound, "couldn't get task file")), + }; let (tx, rx) = mpsc::channel(4); tokio::spawn({ @@ -223,7 +218,7 @@ impl VmService for VMServer { } let file_path = PathBuf::new() - .join(self.file_storage.data_dir()) + .join(&self.file_data_dir) .join(task_id) .join(path); From 3b59b32fbdb00d33c24666f89f0e3fe371c12f11 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 13 Feb 2024 11:02:46 +0100 Subject: [PATCH 24/43] pass clippy and tests --- crates/node/src/main.rs | 4 +- crates/node/src/networking/p2p/pea2pea.rs | 320 ++++++------------ crates/node/src/rpc_server/mod.rs | 70 ++-- crates/node/src/scheduler/mod.rs | 1 + crates/node/src/storage/database/postgres.rs | 34 +- .../node/src/txvalidation/download_manager.rs | 2 +- crates/node/src/txvalidation/event.rs | 12 +- crates/node/src/txvalidation/mod.rs | 7 +- crates/node/src/types/transaction.rs | 15 +- crates/node/src/workflow/mod.rs | 72 ++-- 10 files changed, 225 insertions(+), 312 deletions(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 122ef9b9..141c06eb 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -223,7 +223,7 @@ async fn run(config: Arc) -> Result<()> { let workflow_engine = Arc::new(WorkflowEngine::new(database.clone())); let download_url_prefix = format!( "http://{}:{}", - config.p2p_listen_addr.ip().to_string(), + config.p2p_listen_addr.ip(), config.http_download_port ); @@ -303,7 +303,7 @@ async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { //Indicate some domain conflict issue. //P2P network should be started (peer domain) without Tx management (Node domain) let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel(); - tokio::spawn(async move { while let Some(_) = rcv_tx_event_rx.recv().await {} }); + tokio::spawn(async move { while rcv_tx_event_rx.recv().await.is_some() {} }); let (_, p2p_recv) = mpsc::unbounded_channel::>(); let p2p_stream = UnboundedReceiverStream::new(p2p_recv); diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index 0d22ba96..40207db3 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -26,25 +26,6 @@ use pea2pea::{ }; use sha3::{Digest, Sha3_256}; -// #[async_trait::async_trait] -// pub trait TxHandler: Send + Sync { -// async fn recv_tx(&self, tx: Transaction) -> Result<()>; -// } - -// #[async_trait::async_trait] -// pub trait TxChannel: Send + Sync { -// async fn send_tx(&self, tx: &Transaction) -> Result<()>; -// } - -// struct BlackholeTxHandler; -// #[async_trait::async_trait] -// impl TxHandler for BlackholeTxHandler { -// async fn recv_tx(&self, tx: Transaction) -> Result<()> { -// tracing::debug!("submitting received tx to black hole"); -// Ok(()) -// } -// } - // NOTE: This P2P implementation is originally from `pea2pea` Noise handshake example. #[derive(Clone)] pub struct P2P { @@ -74,6 +55,7 @@ impl Pea2Pea for P2P { } impl P2P { + #[allow(clippy::too_many_arguments)] pub async fn new( name: &str, listen_addr: SocketAddr, @@ -146,24 +128,11 @@ impl P2P { instance } - // pub async fn register_tx_handler(&self, tx_handler: Arc) { - // let mut old_handler = self.tx_handler.write().await; - // *old_handler = tx_handler; - // tracing::debug!("new tx handler registered"); - // } - async fn recv_tx(&self, tx: Transaction) { tracing::debug!("submitting received tx to tx_handler"); if let Err(err) = self.tx_sender.send_tx(tx) { tracing::error!("P2P error during received Tx sending:{err}"); } - - // let tx_handler = self.tx_handler.read().await; - // if let Err(err) = tx_handler.recv_tx(tx).await { - // tracing::error!("failed to handle incoming transaction: {}", err); - // } else { - // tracing::debug!("submitted received tx to tx_handler"); - // } } async fn build_handshake_msg(&self) -> protocol::Handshake { @@ -436,17 +405,6 @@ impl Reading for P2P { } } -// #[async_trait::async_trait] -// impl TxChannel for P2P { -// async fn send_tx(&self, tx: &Transaction) -> Result<()> { -// let msg = protocol::Message::V0(protocol::MessageV0::Transaction(tx.clone())); -// let bs = Bytes::from(bincode::serialize(&msg)?); - -// tracing::debug!("broadcasting transaction {}", tx.hash); -// self.broadcast(bs)?; -// Ok(()) -// } -// } impl Writing for P2P { type Message = Bytes; type Codec = noise::Codec; @@ -473,28 +431,63 @@ impl OnDisconnect for P2P { #[cfg(test)] mod tests { use super::*; + use crate::txvalidation; + use crate::txvalidation::CallbackSender; + use crate::txvalidation::EventProcessError; use eyre::Result; - use gevulot_node::types::{transaction::Payload, Hash, Transaction}; + use gevulot_node::types::transaction::TxReceive; + use gevulot_node::types::{transaction::Payload, Transaction}; use libsecp256k1::SecretKey; - use rand::{rngs::StdRng, RngCore, SeedableRng}; - use tokio::sync::mpsc::{self, Sender}; + use rand::{rngs::StdRng, SeedableRng}; + use tokio::sync::mpsc::UnboundedReceiver; + use tokio::sync::mpsc::UnboundedSender; + use tokio::sync::mpsc::{self}; + use tokio::sync::oneshot; + use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::level_filters::LevelFilter; use tracing_subscriber::EnvFilter; - struct Sink(Arc>); - impl Sink { - fn new(tx: Arc>) -> Self { - Self(tx) - } + async fn create_peer( + name: &str, + ) -> ( + P2P, + UnboundedSender>, + UnboundedReceiver<( + Transaction, + Option>>, + )>, + ) { + let http_peer_list1: Arc>>> = + Default::default(); + let (tx_sender, p2p_recv1) = mpsc::unbounded_channel::>(); + let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv1); + let (sendtx1, txreceiver1) = + mpsc::unbounded_channel::<(Transaction, Option)>(); + let txsender1 = txvalidation::TxEventSender::::build(sendtx1); + let peer = P2P::new( + name, + "127.0.0.1:0".parse().unwrap(), + "secret passphrase", + None, + None, + http_peer_list1, + txsender1, + p2p_stream1, + ) + .await; + (peer, tx_sender, txreceiver1) } - #[async_trait::async_trait] - impl TxHandler for Sink { - async fn recv_tx(&self, tx: Transaction) -> Result<()> { - tracing::debug!("sink received new transaction"); - self.0.send(tx).await.expect("sink send"); - tracing::debug!("sink submitted tx to channel"); - Ok(()) + fn into_receive(tx: Transaction) -> Transaction { + Transaction { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + propagated: tx.executed, + executed: tx.executed, + state: TxReceive::P2P, } } @@ -502,75 +495,15 @@ mod tests { async fn test_peer_list_inter_connection() { //start_logger(LevelFilter::ERROR); - let (tx1, mut rx1) = mpsc::channel(1); - let (tx2, mut rx2) = mpsc::channel(1); - let (tx3, mut rx3) = mpsc::channel(1); - let (sink1, sink2, sink3) = ( - Arc::new(Sink::new(Arc::new(tx1))), - Arc::new(Sink::new(Arc::new(tx2))), - Arc::new(Sink::new(Arc::new(tx3))), - ); - let http_peer_list1: Arc>>> = - Default::default(); - let (_, p2p_recv1) = mpsc::unbounded_channel::(); - let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv); - - let http_peer_list2: Arc>>> = - Default::default(); - let (_, p2p_recv2) = mpsc::unbounded_channel::(); - let p2p_stream2 = UnboundedReceiverStream::new(p2p_recv); - - let http_peer_list3: Arc>>> = - Default::default(); - let (_, p2p_recv3) = mpsc::unbounded_channel::(); - let p2p_stream3 = UnboundedReceiverStream::new(p2p_recv); - - let (peer1, peer2, peer3) = ( - P2P::new( - "peer1", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - None, - None, - http_peer_list1, - p2p_recv1, - p2p_stream1, - ) - .await, - P2P::new( - "peer2", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - Some(9995), - None, - http_peer_list2, - p2p_recv2, - p2p_stream2, - ) - .await, - P2P::new( - "peer3", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - Some(9995), - None, - http_peer_list3, - p2p_recv3, - p2p_stream3, - ) - .await, - ); + let (peer1, tx_sender1, mut tx_receiver1) = create_peer("peer1").await; + let (peer2, tx_sender2, mut tx_receiver2) = create_peer("peer2").await; + let (peer3, tx_sender3, mut tx_receiver3) = create_peer("peer3").await; tracing::debug!("start listening"); let bind_add = peer1.node().start_listening().await.expect("peer1 listen"); let bind_add = peer2.node().start_listening().await.expect("peer2 listen"); let bind_add = peer3.node().start_listening().await.expect("peer3 listen"); - tracing::debug!("register tx handlers"); - peer1.register_tx_handler(sink1.clone()).await; - peer2.register_tx_handler(sink2.clone()).await; - peer3.register_tx_handler(sink3.clone()).await; - tracing::debug!("connect peer2 to peer1"); peer2 .node() @@ -596,75 +529,36 @@ mod tests { tracing::debug!("send tx from peer2 to peer1 and peer3"); let tx = new_tx(); - peer2.send_tx(&tx).await.unwrap(); + tx_sender2.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer1 from peer2"); - let recv_tx = rx1.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver1.recv().await.expect("peer1 recv"); + + assert_eq!(into_receive(tx.clone()), recv_tx.0); tracing::debug!("recv tx on peer3 from peer2"); - let recv_tx = rx3.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver3.recv().await.expect("peer3 recv"); + assert_eq!(into_receive(tx), recv_tx.0); let tx = new_tx(); tracing::debug!("send tx from peer3 to peer1 and peer2"); - peer3.send_tx(&tx).await.unwrap(); + tx_sender3.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer1 from peer3"); - let recv_tx = rx1.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver1.recv().await.expect("peer1 recv"); + assert_eq!(into_receive(tx.clone()), recv_tx.0); tracing::debug!("recv tx on peer2 from peer3"); - let recv_tx = rx2.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver2.recv().await.expect("peer2 recv"); + assert_eq!(into_receive(tx), recv_tx.0); } #[tokio::test] async fn test_two_peers_disconnect() { //start_logger(LevelFilter::ERROR); - let (tx1, mut rx1) = mpsc::channel(1); - let (tx2, mut rx2) = mpsc::channel(1); - let (sink1, sink2) = ( - Arc::new(Sink::new(Arc::new(tx1))), - Arc::new(Sink::new(Arc::new(tx2))), - ); - let http_peer_list1: Arc>>> = - Default::default(); - let (_, p2p_recv1) = mpsc::unbounded_channel::(); - let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv); - - let http_peer_list2: Arc>>> = - Default::default(); - let (_, p2p_recv2) = mpsc::unbounded_channel::(); - let p2p_stream2 = UnboundedReceiverStream::new(p2p_recv); - - let peer1 = P2P::new( - "peer1", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - None, - None, - http_peer_list1, - http_peer_list1, - p2p_recv1, - ) - .await; + let (peer1, tx_sender1, mut tx_receiver1) = create_peer("peer1").await; peer1.node().start_listening().await.expect("peer1 listen"); - peer1.register_tx_handler(sink1.clone()).await; { - let peer2 = P2P::new( - "peer2", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - Some(8776), - None, - http_peer_list2, - http_peer_list2, - p2p_recv2, - ) - .await; + let (peer2, tx_sender2, mut tx_receiver2) = create_peer("peer2").await; peer2.node().start_listening().await.expect("peer2 listen"); - peer2.register_tx_handler(sink2.clone()).await; - - tracing::debug!("Nodes init Done"); peer1 .node() @@ -677,17 +571,17 @@ mod tests { tracing::debug!("Nodes Connected"); tracing::debug!("send tx from peer1 to peer2"); let tx = new_tx(); - peer1.send_tx(&tx).await.unwrap(); + tx_sender1.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer2 from peer1"); - let recv_tx = rx2.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver2.recv().await.expect("peer2 recv"); + assert_eq!(into_receive(tx), recv_tx.0); let tx = new_tx(); tracing::debug!("send tx from peer2 to peer1"); - peer2.send_tx(&tx).await.unwrap(); + tx_sender2.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer1 from peer2"); - let recv_tx = rx1.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver1.recv().await.expect("peer1 recv"); + assert_eq!(into_receive(tx), recv_tx.0); let peers = peer2.node().connected_addrs(); for addr in peers { @@ -700,8 +594,7 @@ mod tests { // Simulate the silent node disconnection by dropping the node. tracing::debug!("send tx from peer1 to disconnected peer2"); let tx = new_tx(); - peer1.send_tx(&tx).await.unwrap(); - + tx_sender1.send(tx).unwrap(); tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; assert_eq!(peer1.peer_list.read().await.len(), 1); @@ -714,39 +607,13 @@ mod tests { async fn test_two_peers() { //start_logger(LevelFilter::ERROR); - let (tx1, mut rx1) = mpsc::channel(1); - let (tx2, mut rx2) = mpsc::channel(1); - let (sink1, sink2) = ( - Arc::new(Sink::new(Arc::new(tx1))), - Arc::new(Sink::new(Arc::new(tx2))), - ); - let (peer1, peer2) = ( - P2P::new( - "peer1", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - None, - None, - ) - .await, - P2P::new( - "peer2", - "127.0.0.1:0".parse().unwrap(), - "secret passphrase", - None, - None, - ) - .await, - ); + let (peer1, tx_sender1, mut tx_receiver1) = create_peer("peer1").await; + let (peer2, tx_sender2, mut tx_receiver2) = create_peer("peer2").await; tracing::debug!("start listening"); peer1.node().start_listening().await.expect("peer1 listen"); peer2.node().start_listening().await.expect("peer2 listen"); - tracing::debug!("register tx handlers"); - peer1.register_tx_handler(sink1.clone()).await; - peer2.register_tx_handler(sink2.clone()).await; - tracing::debug!("connect peer2 to peer1"); peer2 .node() @@ -756,31 +623,34 @@ mod tests { tracing::debug!("send tx from peer1 to peer2"); let tx = new_tx(); - peer1.send_tx(&tx).await.unwrap(); + tx_sender1.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer2 from peer1"); - let recv_tx = rx2.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver2.recv().await.expect("peer2 recv"); + assert_eq!(into_receive(tx), recv_tx.0); let tx = new_tx(); tracing::debug!("send tx from peer2 to peer1"); - peer2.send_tx(&tx).await.unwrap(); + tx_sender2.send(tx.clone()).unwrap(); tracing::debug!("recv tx on peer1 from peer2"); - let recv_tx = rx1.recv().await.expect("sink recv"); - assert_eq!(tx, recv_tx); + let recv_tx = tx_receiver1.recv().await.expect("peer1 recv"); + assert_eq!(into_receive(tx), recv_tx.0); } - fn new_tx() -> Transaction { + fn new_tx() -> Transaction { let rng = &mut StdRng::from_entropy(); - let mut tx = Transaction { - hash: Hash::random(rng), - payload: Payload::Empty, - nonce: rng.next_u64(), - ..Default::default() - }; - let key = SecretKey::random(rng); - tx.sign(&key); - tx + let tx = Transaction::::new(Payload::Empty, &SecretKey::random(rng)); + + Transaction { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + propagated: tx.executed, + executed: tx.executed, + state: TxValdiated, + } } fn start_logger(default_level: LevelFilter) { diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index 3a6c192d..dac2dfd0 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -193,25 +193,28 @@ fn build_tx_tree(hash: &Hash, txs: Vec<(Hash, Option)>) -> Rc RpcServer { + async fn new_rpc_server() -> ( + RpcServer, + UnboundedReceiver<( + Transaction, + Option>>, + )>, + ) { let cfg = Arc::new(Config { data_directory: temp_dir(), db_url: "postgres://gevulot:gevulot@localhost/gevulot".to_string(), @@ -419,27 +443,17 @@ mod tests { http_download_port: 0, }); - let acl_whitelist = Arc::new(AlwaysGrantAclWhitelist {}); let db = Arc::new(Database::new(&cfg.db_url).await.unwrap()); - let mempool = Arc::new(RwLock::new( - Mempool::new(db.clone(), acl_whitelist.clone(), None) - .await - .unwrap(), - )); - let asset_manager = Arc::new(AssetManager::new( - cfg.clone(), - db.clone(), - Arc::new(RwLock::new(std::collections::HashMap::new())), - )); - RpcServer::run( - cfg.clone(), - db.clone(), - mempool, - asset_manager, - acl_whitelist, + let (sendtx, txreceiver) = + mpsc::unbounded_channel::<(Transaction, Option)>(); + let txsender = txvalidation::TxEventSender::::build(sendtx); + + ( + RpcServer::run(cfg, db, txsender) + .await + .expect("rpc_server.run"), + txreceiver, ) - .await - .expect("rpc_server.run") } } diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index e65995f3..463ec593 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -74,6 +74,7 @@ pub struct Scheduler { } impl Scheduler { + #[allow(clippy::too_many_arguments)] pub fn new( mempool: Arc>, database: Arc, diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index 8da188a7..f14a10d5 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -158,7 +158,7 @@ impl Database { b.push_bind(t.id) .push_bind(&new_file.name) .push_bind(&new_file.url) - .push_bind(&new_file.checksum); + .push_bind(new_file.checksum); }); let query = query_builder.build(); @@ -598,7 +598,7 @@ impl Database { b.push_bind(tx.hash) .push_bind(&new_file.name) .push_bind(&new_file.url) - .push_bind(&new_file.checksum); + .push_bind(new_file.checksum); }); let query = query_builder.build(); @@ -643,7 +643,7 @@ impl Database { b.push_bind(tx.hash) .push_bind(&new_file.name) .push_bind(&new_file.url) - .push_bind(&new_file.checksum); + .push_bind(new_file.checksum); }); let query = query_builder.build(); @@ -721,6 +721,7 @@ impl Database { #[cfg(test)] mod tests { + use gevulot_node::types::transaction::TxCreate; use libsecp256k1::{PublicKey, SecretKey}; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -731,6 +732,19 @@ mod tests { use super::*; + fn into_validated(tx: Transaction) -> Transaction { + Transaction { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + propagated: tx.executed, + executed: tx.executed, + state: TxValdiated, + } + } + #[ignore] #[tokio::test] async fn test_add_and_find_deploy_transaction() { @@ -774,6 +788,7 @@ mod tests { signature: Signature::default(), propagated: false, executed: false, + state: TxValdiated, }; database @@ -835,6 +850,7 @@ mod tests { parent: run_tx.hash, prover: prover.hash, proof: vec![1], + files: vec![], }, &key, ); @@ -843,6 +859,7 @@ mod tests { parent: run_tx.hash, prover: prover.hash, proof: vec![2], + files: vec![], }, &key, ); @@ -851,6 +868,7 @@ mod tests { parent: run_tx.hash, prover: prover.hash, proof: vec![3], + files: vec![], }, &key, ); @@ -859,6 +877,7 @@ mod tests { parent: run_tx.hash, prover: prover.hash, proof: vec![4], + files: vec![], }, &key, ); @@ -868,6 +887,7 @@ mod tests { parent: proof1_tx.hash, verifier: verifier.hash, verification: vec![1], + files: vec![], }, &key, ); @@ -876,6 +896,7 @@ mod tests { parent: proof2_tx.hash, verifier: verifier.hash, verification: vec![2], + files: vec![], }, &key, ); @@ -884,6 +905,7 @@ mod tests { parent: proof3_tx.hash, verifier: verifier.hash, verification: vec![3], + files: vec![], }, &key, ); @@ -892,6 +914,7 @@ mod tests { parent: proof4_tx.hash, verifier: verifier.hash, verification: vec![4], + files: vec![], }, &key, ); @@ -923,7 +946,10 @@ mod tests { .expect("add verifier"); for tx in &txs { - database.add_transaction(tx).await.expect("add transaction"); + database + .add_transaction(&into_validated(tx.clone())) + .await + .expect("add transaction"); } // Pick random transaction from set. diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 4ba64309..7af1326a 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -45,7 +45,7 @@ pub async fn download_asset_file( let mut url = reqwest::Url::parse("http://localhost").unwrap(); //unwrap always succeed url.set_ip_host(peer.ip()).unwrap(); //unwrap always succeed url.set_port(Some(port)).unwrap(); //unwrap always succeed - url.set_path(&local_relative_file_path.to_str().unwrap()); //unwrap Path alway ok + url.set_path(local_relative_file_path.to_str().unwrap()); //unwrap Path always ok url }) }) diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index 97621d5a..86484079 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -10,7 +10,7 @@ use futures_util::TryFutureExt; use gevulot_node::types::transaction::AclWhitelist; use std::fmt::Debug; use std::net::SocketAddr; -use std::path::PathBuf; +use std::path::Path; use tokio::sync::mpsc::UnboundedSender; //event type. @@ -35,10 +35,7 @@ pub struct EventTx { impl From> for EventTx { fn from(tx: Transaction) -> Self { - EventTx { - tx: tx, - tx_type: RcvTx, - } + EventTx { tx, tx_type: RcvTx } } } @@ -109,7 +106,7 @@ impl EventTx { impl EventTx { pub async fn process_event( self, - local_directory_path: &PathBuf, + local_directory_path: &Path, http_peer_list: Vec<(SocketAddr, Option)>, ) -> Result<(EventTx, Option>), EventProcessError> { let http_client = reqwest::Client::new(); @@ -133,7 +130,6 @@ impl EventTx { join_all(futures) .await .into_iter() - .map(|res| res) .collect::, _>>() .map_err(|err| { tracing::error!("Error during Tx file download:{err}"); @@ -163,7 +159,7 @@ impl EventTx { state: TxValdiated, }; tracing::info!("Tx validation propagate tx:{}", tx.hash.to_string()); - p2p_sender.send(tx).map_err(|err| err.into()) + p2p_sender.send(tx).map_err(|err| Box::new(err).into()) } } diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 1aa2ecf7..40053eab 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -24,6 +24,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; mod download_manager; mod event; +#[allow(clippy::enum_variant_names)] #[derive(Error, Debug)] pub enum EventProcessError { #[error("Fail to rcv Tx from the channel: {0}")] @@ -34,7 +35,7 @@ pub enum EventProcessError { tokio::sync::mpsc::error::SendError<(Transaction, Option)>, ), #[error("Fail to send the Tx on the channel: {0}")] - PropagateTxError(#[from] tokio::sync::mpsc::error::SendError>), + PropagateTxError(#[from] Box>>), #[error("validation fail: {0}")] ValidateError(String), #[error("Tx asset fail to download because {0}")] @@ -92,7 +93,7 @@ impl TxEventSender { let (sender, rx) = oneshot::channel(); self.sender .send((tx.into_received(TxReceive::RPC), Some(sender))) - .map_err(|err| EventProcessError::from(err))?; + .map_err(EventProcessError::from)?; rx.await? } } @@ -112,7 +113,7 @@ impl TxEventSender { let (sender, rx) = oneshot::channel(); self.sender .send((tx.into_received(TxReceive::TXRESULT), Some(sender))) - .map_err(|err| EventProcessError::from(err))?; + .map_err(EventProcessError::from)?; rx.await? } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index a6db2ee8..9814bb24 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -280,10 +280,7 @@ pub enum TxReceive { impl TxReceive { fn is_from_tx_exec_result(&self) -> bool { - match self { - TxReceive::TXRESULT => true, - _ => false, - } + matches!(self, TxReceive::TXRESULT) } } @@ -515,10 +512,12 @@ mod tests { }; let sk = SecretKey::random(&mut StdRng::from_entropy()); - let mut tx = Transaction::::default(); - tx.author = PublicKey::from_secret_key(&sk); - tx.payload = Payload::Run { workflow }; - tx.signature = Signature::default(); + let tx = Transaction:: { + author: PublicKey::from_secret_key(&sk), + payload: Payload::Run { workflow }, + signature: Signature::default(), + ..Default::default() + }; let tx = tx.into_received(TxReceive::RPC); assert!(tx.validate().is_err()); diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index fb6b1e02..0c0b25b7 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -35,15 +35,11 @@ pub trait TransactionStore: Sync + Send { pub struct WorkflowEngine { tx_store: Arc, - // file_storage: Arc, } impl WorkflowEngine { pub fn new(tx_store: Arc) -> Self { - WorkflowEngine { - tx_store, - // file_storage, - } + WorkflowEngine { tx_store } } pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { @@ -298,7 +294,8 @@ impl WorkflowEngine { #[cfg(test)] mod tests { - use std::{collections::HashMap, env::temp_dir}; + use gevulot_node::types::transaction::TxCreate; + use std::collections::HashMap; use gevulot_node::types::{ transaction::{Payload, ProgramData, Workflow, WorkflowStep}, @@ -310,11 +307,11 @@ mod tests { use super::*; pub struct TxStore { - pub txs: HashMap, + pub txs: HashMap>, } impl TxStore { - pub fn new(txs: &[Transaction]) -> Self { + pub fn new(txs: &[Transaction]) -> Self { let mut store = TxStore { txs: HashMap::with_capacity(txs.len()), }; @@ -327,17 +324,17 @@ mod tests { #[async_trait] impl TransactionStore for TxStore { - async fn find_transaction(&self, tx_hash: &Hash) -> Result> { + async fn find_transaction( + &self, + tx_hash: &Hash, + ) -> Result>> { Ok(self.txs.get(tx_hash).cloned()) } } #[tokio::test] async fn test_next_task_for_empty_workflow_steps() { - let wfe = WorkflowEngine::new( - Arc::new(TxStore::new(&[])), - Arc::new(storage::File::new(&temp_dir())), - ); + let wfe = WorkflowEngine::new(Arc::new(TxStore::new(&[]))); let tx = transaction_for_workflow_steps(vec![]); if let Payload::Run { workflow } = &tx.payload { let res = wfe.next_task(&tx).await; @@ -366,10 +363,7 @@ mod tests { }; let tx = transaction_for_workflow_steps(vec![proving.clone(), verifying]); - let wfe = WorkflowEngine::new( - Arc::new(TxStore::new(&[tx.clone()])), - Arc::new(storage::File::new(&temp_dir())), - ); + let wfe = WorkflowEngine::new(Arc::new(TxStore::new(&[tx.clone()]))); if let Payload::Run { workflow } = &tx.payload { let task = wfe.next_task(&tx).await.expect("next_task").unwrap(); @@ -407,57 +401,69 @@ mod tests { let proofkey_tx = transaction_for_proofkey(&proof_tx.hash); let verification_tx = transaction_for_verification(&proof_tx.hash, &verifier_hash); let tx_store = TxStore::new(&[root_tx, proof_tx, proofkey_tx.clone(), verification_tx]); - let wfe = WorkflowEngine::new( - Arc::new(tx_store), - Arc::new(storage::File::new(&temp_dir())), - ); + let wfe = WorkflowEngine::new(Arc::new(tx_store)); let task = wfe.next_task(&proofkey_tx).await; assert!(task.is_ok()); } - fn transaction_for_workflow_steps(steps: Vec) -> Transaction { + fn into_validated(tx: Transaction) -> Transaction { + Transaction { + author: tx.author, + hash: tx.hash, + payload: tx.payload, + nonce: tx.nonce, + signature: tx.signature, + propagated: tx.executed, + executed: tx.executed, + state: TxValdiated, + } + } + + fn transaction_for_workflow_steps(steps: Vec) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); - Transaction::new( + into_validated(Transaction::new( Payload::Run { workflow: Workflow { steps }, }, &key, - ) + )) } - fn transaction_for_proof(parent: &Hash, program: &Hash) -> Transaction { + fn transaction_for_proof(parent: &Hash, program: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); - Transaction::new( + into_validated(Transaction::new( Payload::Proof { parent: *parent, prover: *program, proof: "proof.".into(), + files: vec![], }, &key, - ) + )) } - fn transaction_for_proofkey(parent: &Hash) -> Transaction { + fn transaction_for_proofkey(parent: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); - Transaction::new( + into_validated(Transaction::new( Payload::ProofKey { parent: *parent, key: "key.".into(), }, &key, - ) + )) } - fn transaction_for_verification(parent: &Hash, program: &Hash) -> Transaction { + fn transaction_for_verification(parent: &Hash, program: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); - Transaction::new( + into_validated(Transaction::new( Payload::Verification { parent: *parent, verifier: *program, verification: b"verification.".to_vec(), + files: vec![], }, &key, - ) + )) } } From 803cfe7d71f32282b09d9253e3cb2d4e36ecf526 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 13 Feb 2024 13:18:25 +0100 Subject: [PATCH 25/43] correct clippy --- crates/shim/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/shim/src/lib.rs b/crates/shim/src/lib.rs index c56f2c53..809f6ead 100644 --- a/crates/shim/src/lib.rs +++ b/crates/shim/src/lib.rs @@ -224,7 +224,7 @@ impl GRPCClient { result: Some(grpc::task_result_request::Result::Task(grpc::TaskResult { id: result.id.clone(), data: result.data.clone(), - files: files, + files, })), }; From a9a4d87affdbb7a63c2bfcc955395508df4e3933 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 14 Feb 2024 10:36:22 +0100 Subject: [PATCH 26/43] do some cleaning --- crates/node/src/main.rs | 5 ---- crates/node/src/mempool/mod.rs | 25 ++----------------- crates/node/src/networking/p2p/pea2pea.rs | 4 --- .../node/src/txvalidation/download_manager.rs | 3 --- crates/shim/src/lib.rs | 3 --- 5 files changed, 2 insertions(+), 38 deletions(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 141c06eb..08e7dc02 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -194,7 +194,6 @@ async fn run(config: Arc) -> Result<()> { http_peer_list, txvalidation::TxEventSender::::build(tx_sender.clone()), p2p_stream, - //database.clone() ) .await, ); @@ -227,8 +226,6 @@ async fn run(config: Arc) -> Result<()> { config.http_download_port ); - //TODO the exec result tx sender has to be added. - //TODO remove mempool. let scheduler = Arc::new(scheduler::Scheduler::new( mempool.clone(), database.clone(), @@ -279,8 +276,6 @@ async fn run(config: Arc) -> Result<()> { let rpc_server = rpc_server::RpcServer::run( config.clone(), database.clone(), - // mempool.clone(), - // asset_mgr.clone(), txvalidation::TxEventSender::::build(tx_sender), ) .await?; diff --git a/crates/node/src/mempool/mod.rs b/crates/node/src/mempool/mod.rs index 35d9c134..7eaeac7c 100644 --- a/crates/node/src/mempool/mod.rs +++ b/crates/node/src/mempool/mod.rs @@ -22,27 +22,15 @@ pub enum MempoolError { #[derive(Clone)] pub struct Mempool { storage: Arc, - // acl_whitelist: Arc, - // TODO: This should be refactored to PubSub channel abstraction later on. - // tx_chan: Option>, deque: VecDeque>, } impl Mempool { - pub async fn new( - storage: Arc, - // acl_whitelist: Arc, - // tx_chan: Option>, - ) -> Result { + pub async fn new(storage: Arc) -> Result { let mut deque = VecDeque::new(); storage.fill_deque(&mut deque).await?; - Ok(Self { - storage, - // acl_whitelist, - // tx_chan, - deque, - }) + Ok(Self { storage, deque }) } pub fn next(&mut self) -> Option> { @@ -64,12 +52,3 @@ impl Mempool { self.deque.len() } } - -// pub struct P2PTxHandler(Arc>); - -// #[async_trait::async_trait] -// impl networking::p2p::TxHandler for P2PTxHandler { -// async fn recv_tx(&self, tx: Transaction) -> Result<()> { -// self.0.write().await.add(tx).await -// } -// } diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index 40207db3..2666f906 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -31,7 +31,6 @@ use sha3::{Digest, Sha3_256}; pub struct P2P { node: Node, noise_states: Arc>>, - //tx_handler: Arc>>, // Peer connection map: <(P2P TCP connection's peer address) , (peer's advertised address in peer_list)>. // This mapping is needed for proper cleanup on OnDisconnect. @@ -287,8 +286,6 @@ impl Handshake for P2P { handshake_msg.peers.insert(handshake_msg.my_p2p_listen_addr); } - tracing::debug!("node information exchanged."); - tracing::debug!("tcp connection peer address: {}", remote_peer); tracing::debug!( "peer advertised address: {}", @@ -338,7 +335,6 @@ impl Handshake for P2P { local_diff.remove(&local_p2p_addr); local_diff.remove(remote_peer_p2p_addr); - tracing::debug!("found {} new nodes", local_diff.len()); let node = self.node(); for addr in local_diff { tracing::debug!("connect to {}", &addr); diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 7af1326a..308f66bc 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -20,9 +20,7 @@ use tokio_util::io::ReaderStream; /// download_file downloads file from the given `url` and saves it to file in `local_directory_path` + / + `file`. pub async fn download_asset_file( - // url: &str, local_directory_path: &Path, - // file: &str, http_peer_list: &[(SocketAddr, Option)], http_client: &reqwest::Client, asset_file: File, @@ -143,7 +141,6 @@ pub async fn serve_files( http_download_port: u16, data_directory: Arc, ) -> Result> { - // let mut bind_addr = config.p2p_listen_addr; bind_addr.set_port(http_download_port); let listener = TcpListener::bind(bind_addr).await?; diff --git a/crates/shim/src/lib.rs b/crates/shim/src/lib.rs index 809f6ead..d405f551 100644 --- a/crates/shim/src/lib.rs +++ b/crates/shim/src/lib.rs @@ -216,9 +216,6 @@ impl GRPCClient { }) }) .collect::>>()?; - // for file in &result.files { - // let checksum = self.submit_file(result.id.clone(), file.clone())?; - // } let task_result_req = grpc::TaskResultRequest { result: Some(grpc::task_result_request::Result::Task(grpc::TaskResult { From 3a1f086f2d4fcdec05c8416ffbc808b0478e0ee8 Mon Sep 17 00:00:00 2001 From: musitdev Date: Fri, 16 Feb 2024 11:35:55 +0100 Subject: [PATCH 27/43] correct PR remarks --- crates/cli/src/lib.rs | 4 +- .../migrations/20231009111925_tasks-table.sql | 1 - .../20231128120351_transactions-table.sql | 13 +--- ...20240215174342_add_txfile_and_checksum.sql | 16 ++++ crates/node/src/cli.rs | 2 +- crates/node/src/main.rs | 22 +++--- crates/node/src/mempool/mod.rs | 16 ++-- crates/node/src/networking/p2p/pea2pea.rs | 40 +++++----- crates/node/src/networking/p2p/protocol.rs | 2 +- crates/node/src/rpc_client/mod.rs | 8 +- crates/node/src/rpc_server/mod.rs | 14 ++-- .../storage/database/entity/transaction.rs | 10 +-- crates/node/src/storage/database/postgres.rs | 37 +++++----- crates/node/src/txvalidation/acl.rs | 24 ++++++ crates/node/src/txvalidation/event.rs | 74 ++++++++++--------- crates/node/src/txvalidation/mod.rs | 52 ++++++------- crates/node/src/types/file.rs | 8 +- crates/node/src/types/transaction.rs | 58 ++++++--------- crates/node/src/vmm/qemu.rs | 2 +- crates/node/src/workflow/mod.rs | 29 ++++---- crates/shim/proto/vm_service.proto | 2 +- crates/tests/e2e-tests/src/main.rs | 4 +- 22 files changed, 224 insertions(+), 214 deletions(-) create mode 100644 crates/node/migrations/20240215174342_add_txfile_and_checksum.sql create mode 100644 crates/node/src/txvalidation/acl.rs diff --git a/crates/cli/src/lib.rs b/crates/cli/src/lib.rs index 474cc05e..4bea57ff 100644 --- a/crates/cli/src/lib.rs +++ b/crates/cli/src/lib.rs @@ -1,5 +1,5 @@ use crate::file::FileData; -use gevulot_node::types::transaction::{ProgramMetadata, TxCreate}; +use gevulot_node::types::transaction::{Created, ProgramMetadata}; use gevulot_node::types::Hash; use gevulot_node::{ rpc_client::RpcClient, @@ -203,7 +203,7 @@ pub async fn run_deploy_command( )) } -async fn send_transaction(client: &RpcClient, tx: &Transaction) -> Result { +async fn send_transaction(client: &RpcClient, tx: &Transaction) -> Result { client .send_transaction(tx) .await diff --git a/crates/node/migrations/20231009111925_tasks-table.sql b/crates/node/migrations/20231009111925_tasks-table.sql index 0ecbd9be..1c2ce845 100644 --- a/crates/node/migrations/20231009111925_tasks-table.sql +++ b/crates/node/migrations/20231009111925_tasks-table.sql @@ -24,7 +24,6 @@ CREATE TABLE file ( task_id uuid NOT NULL, name VARCHAR(256) NOT NULL, url VARCHAR(2048) NOT NULL, - checksum VARCHAR(64) NOT NULL, CONSTRAINT fk_task FOREIGN KEY (task_id) REFERENCES task (id) ON DELETE CASCADE diff --git a/crates/node/migrations/20231128120351_transactions-table.sql b/crates/node/migrations/20231128120351_transactions-table.sql index 34dfbd9c..27f8cc5d 100644 --- a/crates/node/migrations/20231128120351_transactions-table.sql +++ b/crates/node/migrations/20231128120351_transactions-table.sql @@ -116,15 +116,4 @@ CREATE TABLE proof_key ( CONSTRAINT fk_transaction FOREIGN KEY (tx) REFERENCES transaction (hash) ON DELETE CASCADE -); - - -CREATE TABLE txfile ( - tx_id VARCHAR(64) NOT NULL, - name VARCHAR(256) NOT NULL, - url VARCHAR(2048) NOT NULL, - checksum VARCHAR(64) NOT NULL, - CONSTRAINT fk_tx - FOREIGN KEY (tx_id) - REFERENCES transaction (hash) ON DELETE CASCADE -); +); \ No newline at end of file diff --git a/crates/node/migrations/20240215174342_add_txfile_and_checksum.sql b/crates/node/migrations/20240215174342_add_txfile_and_checksum.sql new file mode 100644 index 00000000..e84b5881 --- /dev/null +++ b/crates/node/migrations/20240215174342_add_txfile_and_checksum.sql @@ -0,0 +1,16 @@ +-- Add checksum to task files +-- Add a new table to store Tx files. + +ALTER TABLE file ADD checksum VARCHAR(64) NOT NULL; + +DROP TABLE IF EXISTS txfile; + +CREATE TABLE txfile ( + tx_id VARCHAR(64) NOT NULL, + name VARCHAR(256) NOT NULL, + url VARCHAR(2048) NOT NULL, + checksum VARCHAR(64) NOT NULL, + CONSTRAINT fk_tx + FOREIGN KEY (tx_id) + REFERENCES transaction (hash) ON DELETE CASCADE +); diff --git a/crates/node/src/cli.rs b/crates/node/src/cli.rs index 323901da..e4e862c8 100644 --- a/crates/node/src/cli.rs +++ b/crates/node/src/cli.rs @@ -180,7 +180,7 @@ pub struct P2PBeaconConfig { #[arg( long, - long_help = "Port open to download transaction data between nodes. Use P2P interface to bind.", + long_help = "HTTP port for downloading transaction data between nodes. Uses same interface as P2P listen address.", env = "GEVULOT_HTTP_PORT", default_value = "9995" )] diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 08e7dc02..1a45386f 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -28,7 +28,7 @@ use tokio::sync::{Mutex as TMutex, RwLock}; use tokio_stream::wrappers::UnboundedReceiverStream; use tonic::transport::Server; use tracing_subscriber::{filter::LevelFilter, fmt::format::FmtSpan, EnvFilter}; -use types::{transaction::TxValdiated, Hash, Transaction}; +use types::{transaction::Validated, Hash, Transaction}; use workflow::WorkflowEngine; mod cli; @@ -135,11 +135,11 @@ fn generate_node_key(opts: NodeKeyOptions) -> Result<()> { #[async_trait] impl mempool::Storage for storage::Database { - async fn get(&self, hash: &Hash) -> Result>> { + async fn get(&self, hash: &Hash) -> Result>> { self.find_transaction(hash).await } - async fn set(&self, tx: &Transaction) -> Result<()> { + async fn set(&self, tx: &Transaction) -> Result<()> { let tx_hash = tx.hash; self.add_transaction(tx).await?; self.add_asset(&tx_hash).await?; @@ -148,7 +148,7 @@ impl mempool::Storage for storage::Database { async fn fill_deque( &self, - deque: &mut std::collections::VecDeque>, + deque: &mut std::collections::VecDeque>, ) -> Result<()> { for t in self.get_unexecuted_transactions().await? { deque.push_back(t); @@ -160,7 +160,7 @@ impl mempool::Storage for storage::Database { #[async_trait] impl workflow::TransactionStore for storage::Database { - async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { + async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { self.find_transaction(tx_hash).await } } @@ -173,8 +173,8 @@ async fn run(config: Arc) -> Result<()> { let mempool = Arc::new(RwLock::new(Mempool::new(database.clone()).await?)); - //start Tx process event loop - let (txevent_loop_jh, tx_sender, p2p_stream) = txvalidation::start_event_loop( + // Start Tx process event loop. + let (txevent_loop_jh, tx_sender, p2p_stream) = txvalidation::spawn_event_loop( config.data_directory.clone(), config.p2p_listen_addr, config.http_download_port, @@ -294,13 +294,13 @@ async fn p2p_beacon(config: P2PBeaconConfig) -> Result<()> { let http_peer_list: Arc>>> = Default::default(); - //build empty channel for P2P interface Transaction management. - //Indicate some domain conflict issue. - //P2P network should be started (peer domain) without Tx management (Node domain) + // Build an empty channel for P2P interface's `Transaction` management. + // Indicate some domain conflict issue. + // P2P network should be started (peer domain) without Tx management (Node domain). let (tx, mut rcv_tx_event_rx) = mpsc::unbounded_channel(); tokio::spawn(async move { while rcv_tx_event_rx.recv().await.is_some() {} }); - let (_, p2p_recv) = mpsc::unbounded_channel::>(); + let (_, p2p_recv) = mpsc::unbounded_channel::>(); let p2p_stream = UnboundedReceiverStream::new(p2p_recv); let p2p = Arc::new( diff --git a/crates/node/src/mempool/mod.rs b/crates/node/src/mempool/mod.rs index 7eaeac7c..42be3745 100644 --- a/crates/node/src/mempool/mod.rs +++ b/crates/node/src/mempool/mod.rs @@ -1,4 +1,4 @@ -use crate::types::{transaction::TxValdiated, Hash, Transaction}; +use crate::types::{transaction::Validated, Hash, Transaction}; use async_trait::async_trait; use eyre::Result; use std::collections::VecDeque; @@ -7,9 +7,9 @@ use thiserror::Error; #[async_trait] pub trait Storage: Send + Sync { - async fn get(&self, hash: &Hash) -> Result>>; - async fn set(&self, tx: &Transaction) -> Result<()>; - async fn fill_deque(&self, deque: &mut VecDeque>) -> Result<()>; + async fn get(&self, hash: &Hash) -> Result>>; + async fn set(&self, tx: &Transaction) -> Result<()>; + async fn fill_deque(&self, deque: &mut VecDeque>) -> Result<()>; } #[allow(clippy::enum_variant_names)] @@ -22,7 +22,7 @@ pub enum MempoolError { #[derive(Clone)] pub struct Mempool { storage: Arc, - deque: VecDeque>, + deque: VecDeque>, } impl Mempool { @@ -33,16 +33,16 @@ impl Mempool { Ok(Self { storage, deque }) } - pub fn next(&mut self) -> Option> { + pub fn next(&mut self) -> Option> { // TODO(tuommaki): Should storage reflect the POP in state? self.deque.pop_front() } - pub fn peek(&self) -> Option<&Transaction> { + pub fn peek(&self) -> Option<&Transaction> { self.deque.front() } - pub async fn add(&mut self, tx: Transaction) -> Result<()> { + pub async fn add(&mut self, tx: Transaction) -> Result<()> { self.storage.set(&tx).await?; self.deque.push_back(tx); Ok(()) diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index 2666f906..b979cd25 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -16,7 +16,7 @@ use tokio_stream::StreamExt; use super::{noise, protocol}; use bytes::{Bytes, BytesMut}; use gevulot_node::types::{ - transaction::{TxCreate, TxValdiated}, + transaction::{Created, Validated}, Transaction, }; use parking_lot::RwLock; @@ -36,14 +36,14 @@ pub struct P2P { // This mapping is needed for proper cleanup on OnDisconnect. peer_addr_mapping: Arc>>, peer_list: Arc>>, - //contains corrected peers use for asset file download. + // Contains corrected peers that are used for asset file download. pub peer_http_port_list: Arc>>>, http_port: Option, nat_listen_addr: Option, psk: Vec, - //send Tx to the process loop + // Send Tx to the process loop. tx_sender: TxEventSender, } @@ -63,7 +63,7 @@ impl P2P { nat_listen_addr: Option, peer_http_port_list: Arc>>>, tx_sender: TxEventSender, - propagate_tx_stream: impl Stream> + std::marker::Send + 'static, + propagate_tx_stream: impl Stream> + std::marker::Send + 'static, ) -> Self { let config = Config { name: Some(name.into()), @@ -82,7 +82,6 @@ impl P2P { let instance = Self { node, noise_states: Default::default(), - //tx_handler: Arc::new(tokio::sync::RwLock::new(Arc::new(BlackholeTxHandler {}))), psk: psk.to_vec(), peer_list: Default::default(), peer_addr_mapping: Default::default(), @@ -98,7 +97,7 @@ impl P2P { instance.enable_writing().await; instance.enable_disconnect().await; - //start new Tx stream loop + // Start a new Tx stream loop. tokio::spawn({ let p2p = instance.clone(); async move { @@ -127,7 +126,7 @@ impl P2P { instance } - async fn recv_tx(&self, tx: Transaction) { + async fn forward_tx(&self, tx: Transaction) { tracing::debug!("submitting received tx to tx_handler"); if let Err(err) = self.tx_sender.send_tx(tx) { tracing::error!("P2P error during received Tx sending:{err}"); @@ -375,7 +374,7 @@ impl Reading for P2P { tx.hash, hex::encode(tx.author.serialize()) ); - let tx: Transaction = Transaction { + let tx: Transaction = Transaction { author: tx.author, hash: tx.hash, payload: tx.payload, @@ -383,9 +382,9 @@ impl Reading for P2P { signature: tx.signature, propagated: tx.propagated, executed: tx.executed, - state: TxCreate, + state: Created, }; - self.recv_tx(tx).await; + self.forward_tx(tx).await; } protocol::MessageV0::DiagnosticsRequest(kind) => { tracing::debug!("received diagnostics request"); @@ -431,7 +430,7 @@ mod tests { use crate::txvalidation::CallbackSender; use crate::txvalidation::EventProcessError; use eyre::Result; - use gevulot_node::types::transaction::TxReceive; + use gevulot_node::types::transaction::Received; use gevulot_node::types::{transaction::Payload, Transaction}; use libsecp256k1::SecretKey; use rand::{rngs::StdRng, SeedableRng}; @@ -447,18 +446,18 @@ mod tests { name: &str, ) -> ( P2P, - UnboundedSender>, + UnboundedSender>, UnboundedReceiver<( - Transaction, + Transaction, Option>>, )>, ) { let http_peer_list1: Arc>>> = Default::default(); - let (tx_sender, p2p_recv1) = mpsc::unbounded_channel::>(); + let (tx_sender, p2p_recv1) = mpsc::unbounded_channel::>(); let p2p_stream1 = UnboundedReceiverStream::new(p2p_recv1); let (sendtx1, txreceiver1) = - mpsc::unbounded_channel::<(Transaction, Option)>(); + mpsc::unbounded_channel::<(Transaction, Option)>(); let txsender1 = txvalidation::TxEventSender::::build(sendtx1); let peer = P2P::new( name, @@ -474,7 +473,8 @@ mod tests { (peer, tx_sender, txreceiver1) } - fn into_receive(tx: Transaction) -> Transaction { + //TODO change by impl From when module declaration between main and lib are solved. + fn into_receive(tx: Transaction) -> Transaction { Transaction { author: tx.author, hash: tx.hash, @@ -483,7 +483,7 @@ mod tests { signature: tx.signature, propagated: tx.executed, executed: tx.executed, - state: TxReceive::P2P, + state: Received::P2P, } } @@ -632,10 +632,10 @@ mod tests { assert_eq!(into_receive(tx), recv_tx.0); } - fn new_tx() -> Transaction { + fn new_tx() -> Transaction { let rng = &mut StdRng::from_entropy(); - let tx = Transaction::::new(Payload::Empty, &SecretKey::random(rng)); + let tx = Transaction::::new(Payload::Empty, &SecretKey::random(rng)); Transaction { author: tx.author, @@ -645,7 +645,7 @@ mod tests { signature: tx.signature, propagated: tx.executed, executed: tx.executed, - state: TxValdiated, + state: Validated, } } diff --git a/crates/node/src/networking/p2p/protocol.rs b/crates/node/src/networking/p2p/protocol.rs index 0dae6b68..3afc1df2 100644 --- a/crates/node/src/networking/p2p/protocol.rs +++ b/crates/node/src/networking/p2p/protocol.rs @@ -23,7 +23,7 @@ pub(crate) enum Message { #[allow(clippy::large_enum_variant)] #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) enum MessageV0 { - Transaction(types::Transaction), + Transaction(types::Transaction), DiagnosticsRequest(DiagnosticsRequestKind), DiagnosticsResponse(DiagnosticsResponseV0), } diff --git a/crates/node/src/rpc_client/mod.rs b/crates/node/src/rpc_client/mod.rs index de79e03a..25c257c9 100644 --- a/crates/node/src/rpc_client/mod.rs +++ b/crates/node/src/rpc_client/mod.rs @@ -7,7 +7,7 @@ use jsonrpsee::{ use crate::types::{ rpc::RpcResponse, - transaction::{TransactionTree, TxCreate, TxValdiated}, + transaction::{Created, TransactionTree, Validated}, Hash, Transaction, }; @@ -26,13 +26,13 @@ impl RpcClient { pub async fn get_transaction( &self, tx_hash: &Hash, - ) -> Result>, Box> { + ) -> Result>, Box> { let mut params = ArrayParams::new(); params.insert(tx_hash).expect("rpc params"); let resp = self .client - .request::>, ArrayParams>("getTransaction", params) + .request::>, ArrayParams>("getTransaction", params) .await .expect("rpc request"); @@ -42,7 +42,7 @@ impl RpcClient { } } - pub async fn send_transaction(&self, tx: &Transaction) -> Result<(), Box> { + pub async fn send_transaction(&self, tx: &Transaction) -> Result<(), Box> { let mut params = ArrayParams::new(); params.insert(tx).expect("rpc params"); diff --git a/crates/node/src/rpc_server/mod.rs b/crates/node/src/rpc_server/mod.rs index dac2dfd0..aff03d74 100644 --- a/crates/node/src/rpc_server/mod.rs +++ b/crates/node/src/rpc_server/mod.rs @@ -7,7 +7,7 @@ use crate::{ cli::Config, storage::Database, types::{ - transaction::{TxCreate, TxValdiated}, + transaction::{Created, Validated}, Transaction, }, }; @@ -76,7 +76,7 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp tracing::info!("JSON-RPC: send_transaction()"); // Real logic - let tx: Transaction = match params.one() { + let tx: Transaction = match params.one() { Ok(tx) => tx, Err(e) => { tracing::error!("failed to parse transaction: {}", e); @@ -98,7 +98,7 @@ async fn send_transaction(params: Params<'static>, ctx: Arc) -> RpcResp async fn get_transaction( params: Params<'static>, ctx: Arc, -) -> RpcResponse> { +) -> RpcResponse> { let tx_hash: Hash = match params.one() { Ok(tx_hash) => tx_hash, Err(e) => { @@ -197,7 +197,7 @@ mod tests { use crate::txvalidation; use crate::txvalidation::CallbackSender; use crate::txvalidation::EventProcessError; - use gevulot_node::types::transaction::TxReceive; + use gevulot_node::types::transaction::Received; use jsonrpsee::{ core::{client::ClientT, params::ArrayParams}, http_client::HttpClientBuilder, @@ -243,7 +243,7 @@ mod tests { signature: tx.signature, propagated: tx.executed, executed: tx.executed, - state: TxReceive::RPC, + state: Received::RPC, }; assert_eq!(tx, recv_tx.0); @@ -421,7 +421,7 @@ mod tests { async fn new_rpc_server() -> ( RpcServer, UnboundedReceiver<( - Transaction, + Transaction, Option>>, )>, ) { @@ -446,7 +446,7 @@ mod tests { let db = Arc::new(Database::new(&cfg.db_url).await.unwrap()); let (sendtx, txreceiver) = - mpsc::unbounded_channel::<(Transaction, Option)>(); + mpsc::unbounded_channel::<(Transaction, Option)>(); let txsender = txvalidation::TxEventSender::::build(sendtx); ( diff --git a/crates/node/src/storage/database/entity/transaction.rs b/crates/node/src/storage/database/entity/transaction.rs index 4d6ef8a4..4fca6010 100644 --- a/crates/node/src/storage/database/entity/transaction.rs +++ b/crates/node/src/storage/database/entity/transaction.rs @@ -29,8 +29,8 @@ pub struct Transaction { pub executed: bool, } -impl From<&types::Transaction> for Transaction { - fn from(value: &types::Transaction) -> Self { +impl From<&types::Transaction> for Transaction { + fn from(value: &types::Transaction) -> Self { let kind = match value.payload { transaction::Payload::Empty => Kind::Empty, transaction::Payload::Transfer { .. } => Kind::Transfer, @@ -56,8 +56,8 @@ impl From<&types::Transaction> for Transaction { } } -impl From for types::Transaction { - fn from(value: Transaction) -> types::Transaction { +impl From for types::Transaction { + fn from(value: Transaction) -> types::Transaction { types::Transaction { author: value.author.into(), hash: value.hash, @@ -68,7 +68,7 @@ impl From for types::Transaction { signature: value.signature, propagated: value.propagated, executed: value.executed, - state: transaction::TxValdiated, + state: transaction::Validated, } } } diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index f14a10d5..bd920df4 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -1,14 +1,14 @@ use super::entity::{self}; +use crate::txvalidation::acl::AclWhiteListError; +use crate::txvalidation::acl::AclWhitelist; use crate::types::file::DbFile; -use crate::types::transaction; use crate::types::{ self, - transaction::{ProgramData, TxValdiated}, + transaction::{ProgramData, Validated}, Hash, Program, Task, }; use eyre::Result; use gevulot_node::types::program::ResourceRequest; -use gevulot_node::types::transaction::TransactionError; use libsecp256k1::PublicKey; use sqlx::{self, postgres::PgPoolOptions, FromRow, Row}; use std::time::Duration; @@ -18,12 +18,12 @@ const MAX_DB_CONNS: u32 = 64; const DB_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); #[async_trait::async_trait] -impl transaction::AclWhitelist for Database { - async fn contains(&self, key: &PublicKey) -> Result { +impl AclWhitelist for Database { + async fn contains(&self, key: &PublicKey) -> Result { let key = entity::PublicKey(*key); - self.acl_whitelist_has(&key) - .await - .map_err(|err| TransactionError::General(format!("Fail to query access list: {err}",))) + self.acl_whitelist_has(&key).await.map_err(|err| { + AclWhiteListError::InternalError(format!("Fail to query access list: {err}",)) + }) } } @@ -270,7 +270,7 @@ impl Database { pub async fn find_transaction( &self, tx_hash: &Hash, - ) -> Result>> { + ) -> Result>> { let mut db_tx = self.pool.begin().await?; let entity = @@ -426,7 +426,7 @@ impl Database { _ => types::transaction::Payload::Empty, }; - let mut tx: types::transaction::Transaction = entity.into(); + let mut tx: types::transaction::Transaction = entity.into(); tx.payload = payload; Ok(Some(tx)) } else { @@ -434,7 +434,7 @@ impl Database { } } - pub async fn get_transactions(&self) -> Result>> { + pub async fn get_transactions(&self) -> Result>> { let mut db_tx = self.pool.begin().await?; let refs: Vec = sqlx::query("SELECT hash FROM transaction") .map(|row: sqlx::postgres::PgRow| row.get(0)) @@ -452,9 +452,7 @@ impl Database { Ok(txs) } - pub async fn get_unexecuted_transactions( - &self, - ) -> Result>> { + pub async fn get_unexecuted_transactions(&self) -> Result>> { let mut db_tx = self.pool.begin().await?; let refs: Vec = sqlx::query("SELECT hash FROM transaction WHERE executed IS false") .map(|row: sqlx::postgres::PgRow| row.get(0)) @@ -488,7 +486,7 @@ impl Database { Ok(refs) } - pub async fn add_transaction(&self, tx: &types::Transaction) -> Result<()> { + pub async fn add_transaction(&self, tx: &types::Transaction) -> Result<()> { let entity = entity::Transaction::from(tx); let mut db_tx = self.pool.begin().await?; @@ -721,7 +719,7 @@ impl Database { #[cfg(test)] mod tests { - use gevulot_node::types::transaction::TxCreate; + use gevulot_node::types::transaction::Created; use libsecp256k1::{PublicKey, SecretKey}; use rand::{rngs::StdRng, Rng, SeedableRng}; @@ -732,7 +730,8 @@ mod tests { use super::*; - fn into_validated(tx: Transaction) -> Transaction { + //TODO change by impl From when module declaration between main and lib are solved. + fn into_validated(tx: Transaction) -> Transaction { Transaction { author: tx.author, hash: tx.hash, @@ -741,7 +740,7 @@ mod tests { signature: tx.signature, propagated: tx.executed, executed: tx.executed, - state: TxValdiated, + state: Validated, } } @@ -788,7 +787,7 @@ mod tests { signature: Signature::default(), propagated: false, executed: false, - state: TxValdiated, + state: Validated, }; database diff --git a/crates/node/src/txvalidation/acl.rs b/crates/node/src/txvalidation/acl.rs new file mode 100644 index 00000000..36587f2d --- /dev/null +++ b/crates/node/src/txvalidation/acl.rs @@ -0,0 +1,24 @@ +use async_trait::async_trait; +use eyre::Result; +use libsecp256k1::PublicKey; +use thiserror::Error; + +#[allow(clippy::enum_variant_names)] +#[derive(Error, Debug)] +pub enum AclWhiteListError { + #[error("An error occurs during ACLlist validation: {0}")] + InternalError(String), +} + +#[async_trait] +pub trait AclWhitelist: Send + Sync { + async fn contains(&self, key: &PublicKey) -> Result; +} + +pub struct AlwaysGrantAclWhitelist; +#[async_trait::async_trait] +impl AclWhitelist for AlwaysGrantAclWhitelist { + async fn contains(&self, _key: &PublicKey) -> Result { + Ok(true) + } +} diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index 86484079..b865fc62 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -1,13 +1,13 @@ +use crate::txvalidation::acl::AclWhitelist; use crate::txvalidation::download_manager; use crate::txvalidation::EventProcessError; use crate::types::{ - transaction::{TransactionError, TxReceive, TxValdiated}, + transaction::{Received, Validated}, Transaction, }; use crate::Mempool; use futures::future::join_all; use futures_util::TryFutureExt; -use gevulot_node::types::transaction::AclWhitelist; use std::fmt::Debug; use std::net::SocketAddr; use std::path::Path; @@ -15,7 +15,7 @@ use tokio::sync::mpsc::UnboundedSender; //event type. #[derive(Debug, Clone)] -pub struct RcvTx; +pub struct ReceivedTx; #[derive(Debug, Clone)] pub struct DownloadTx; @@ -28,44 +28,47 @@ pub struct PropagateTx; //Event processing depends on the marker type. #[derive(Debug, Clone)] -pub struct EventTx { - pub tx: Transaction, +pub struct TxEvent { + pub tx: Transaction, pub tx_type: T, } -impl From> for EventTx { - fn from(tx: Transaction) -> Self { - EventTx { tx, tx_type: RcvTx } +impl From> for TxEvent { + fn from(tx: Transaction) -> Self { + TxEvent { + tx, + tx_type: ReceivedTx, + } } } -impl From> for EventTx { - fn from(event: EventTx) -> Self { - EventTx { +impl From> for TxEvent { + fn from(event: TxEvent) -> Self { + TxEvent { tx: event.tx, tx_type: DownloadTx, } } } -impl From> for EventTx { - fn from(event: EventTx) -> Self { - EventTx { +impl From> for TxEvent { + fn from(event: TxEvent) -> Self { + TxEvent { tx: event.tx, tx_type: NewTx, } } } -impl From> for Option> { - fn from(event: EventTx) -> Self { +impl From> for Option> { + fn from(event: TxEvent) -> Self { match event.tx.state { - TxReceive::P2P => None, - TxReceive::RPC => Some(EventTx { + Received::P2P => None, + Received::RPC => Some(TxEvent { tx: event.tx, tx_type: PropagateTx, }), - TxReceive::TXRESULT => Some(EventTx { + Received::TXRESULT => Some(TxEvent { tx: event.tx, tx_type: PropagateTx, }), @@ -74,11 +77,11 @@ impl From> for Option> { } //Processing of event that arrive: SourceTxType. -impl EventTx { +impl TxEvent { pub async fn process_event( self, acl_whitelist: &impl AclWhitelist, - ) -> Result, EventProcessError> { + ) -> Result, EventProcessError> { match self.validate_tx(acl_whitelist).await { Ok(()) => Ok(self.into()), Err(err) => Err(EventProcessError::ValidateError(format!( @@ -88,12 +91,17 @@ impl EventTx { } //Tx validation process. - async fn validate_tx(&self, acl_whitelist: &impl AclWhitelist) -> Result<(), TransactionError> { - self.tx.validate()?; + async fn validate_tx( + &self, + acl_whitelist: &impl AclWhitelist, + ) -> Result<(), EventProcessError> { + self.tx.validate().map_err(|err| { + EventProcessError::ValidateError(format!("Error during transaction validation:{err}",)) + })?; // Secondly verify that author is whitelisted. if !acl_whitelist.contains(&self.tx.author).await? { - return Err(TransactionError::Validation( + return Err(EventProcessError::ValidateError( "Tx permission denied signer not authorized".to_string(), )); } @@ -103,12 +111,12 @@ impl EventTx { } //Download Tx processing -impl EventTx { +impl TxEvent { pub async fn process_event( self, local_directory_path: &Path, http_peer_list: Vec<(SocketAddr, Option)>, - ) -> Result<(EventTx, Option>), EventProcessError> { + ) -> Result<(TxEvent, Option>), EventProcessError> { let http_client = reqwest::Client::new(); let asset_file_list = self.tx.get_asset_list().map_err(|err| { EventProcessError::DownloadAssetError(format!( @@ -135,17 +143,17 @@ impl EventTx { tracing::error!("Error during Tx file download:{err}"); EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) })?; - let newtx: EventTx = self.clone().into(); - let propagate: Option> = self.into(); + let newtx: TxEvent = self.clone().into(); + let propagate: Option> = self.into(); Ok((newtx, propagate)) } } //Propagate Tx processing -impl EventTx { +impl TxEvent { pub async fn process_event( self, - p2p_sender: &UnboundedSender>, + p2p_sender: &UnboundedSender>, ) -> Result<(), EventProcessError> { let tx = Transaction { author: self.tx.author, @@ -156,14 +164,14 @@ impl EventTx { //TODO should be updated after the p2p send with a notification propagated: true, executed: self.tx.executed, - state: TxValdiated, + state: Validated, }; tracing::info!("Tx validation propagate tx:{}", tx.hash.to_string()); p2p_sender.send(tx).map_err(|err| Box::new(err).into()) } } -impl EventTx { +impl TxEvent { pub async fn process_event(self, mempool: &mut Mempool) -> Result<(), EventProcessError> { let tx = Transaction { author: self.tx.author, @@ -174,7 +182,7 @@ impl EventTx { //TODO should be updated after the p2p send with a notification propagated: true, executed: self.tx.executed, - state: TxValdiated, + state: Validated, }; tracing::info!("Tx validation save tx:{}", tx.hash.to_string()); mempool diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 40053eab..8095e7d9 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -1,12 +1,11 @@ -use crate::txvalidation::event::{EventTx, RcvTx}; +use crate::txvalidation::event::{ReceivedTx, TxEvent}; use crate::types::{ - transaction::{TxCreate, TxReceive, TxValdiated}, + transaction::{Created, Received, Validated}, Transaction, }; use crate::Mempool; use futures_util::Stream; use futures_util::TryFutureExt; -use gevulot_node::types::transaction::AclWhitelist; use std::collections::HashMap; use std::fmt::Debug; use std::marker::PhantomData; @@ -21,6 +20,7 @@ use tokio::sync::RwLock; use tokio::task::JoinHandle; use tokio_stream::wrappers::UnboundedReceiverStream; +pub mod acl; mod download_manager; mod event; @@ -32,16 +32,18 @@ pub enum EventProcessError { #[error("Fail to send the Tx on the channel: {0}")] SendChannelError( #[from] - tokio::sync::mpsc::error::SendError<(Transaction, Option)>, + tokio::sync::mpsc::error::SendError<(Transaction, Option)>, ), #[error("Fail to send the Tx on the channel: {0}")] - PropagateTxError(#[from] Box>>), + PropagateTxError(#[from] Box>>), #[error("validation fail: {0}")] ValidateError(String), #[error("Tx asset fail to download because {0}")] DownloadAssetError(String), #[error("Save Tx error: {0}")] SaveTxError(String), + #[error("AclWhite list authenticate error: {0}")] + AclWhiteListAuthError(#[from] acl::AclWhiteListError), } pub type CallbackSender = oneshot::Sender>; @@ -56,43 +58,39 @@ pub struct TxResultSender; //use to send a tx to the event process. #[derive(Debug, Clone)] pub struct TxEventSender { - sender: UnboundedSender<(Transaction, Option)>, + sender: UnboundedSender<(Transaction, Option)>, _marker: PhantomData, } //Manage send from the p2p source impl TxEventSender { - pub fn build( - sender: UnboundedSender<(Transaction, Option)>, - ) -> Self { + pub fn build(sender: UnboundedSender<(Transaction, Option)>) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + pub fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { self.sender - .send((tx.into_received(TxReceive::P2P), None)) + .send((tx.into_received(Received::P2P), None)) .map_err(|err| err.into()) } } //Manage send from the RPC source impl TxEventSender { - pub fn build( - sender: UnboundedSender<(Transaction, Option)>, - ) -> Self { + pub fn build(sender: UnboundedSender<(Transaction, Option)>) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { let (sender, rx) = oneshot::channel(); self.sender - .send((tx.into_received(TxReceive::RPC), Some(sender))) + .send((tx.into_received(Received::RPC), Some(sender))) .map_err(EventProcessError::from)?; rx.await? } @@ -100,40 +98,38 @@ impl TxEventSender { //Manage send from the Tx result execution source impl TxEventSender { - pub fn build( - sender: UnboundedSender<(Transaction, Option)>, - ) -> Self { + pub fn build(sender: UnboundedSender<(Transaction, Option)>) -> Self { TxEventSender { sender, _marker: PhantomData, } } - pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { + pub async fn send_tx(&self, tx: Transaction) -> Result<(), EventProcessError> { let (sender, rx) = oneshot::channel(); self.sender - .send((tx.into_received(TxReceive::TXRESULT), Some(sender))) + .send((tx.into_received(Received::TXRESULT), Some(sender))) .map_err(EventProcessError::from)?; rx.await? } } //Main event processing loog. -pub async fn start_event_loop( +pub async fn spawn_event_loop( local_directory_path: PathBuf, bind_addr: SocketAddr, http_download_port: u16, http_peer_list: Arc>>>, - acl_whitelist: Arc, + acl_whitelist: Arc, //New Tx are added to the mempool directly. //Like for the p2p a stream can be use to decouple both process. mempool: Arc>, ) -> eyre::Result<( JoinHandle<()>, //channel use to send RcvTx event to the processing - UnboundedSender<(Transaction, Option)>, + UnboundedSender<(Transaction, Option)>, //output stream use to propagate Tx. - impl Stream>, + impl Stream>, )> { let local_directory_path = Arc::new(local_directory_path); //start http download manager @@ -142,9 +138,9 @@ pub async fn start_event_loop( .await?; let (tx, mut rcv_tx_event_rx) = - mpsc::unbounded_channel::<(Transaction, Option)>(); + mpsc::unbounded_channel::<(Transaction, Option)>(); - let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::>(); + let (p2p_sender, p2p_recv) = mpsc::unbounded_channel::>(); let p2p_stream = UnboundedReceiverStream::new(p2p_recv); let jh = tokio::spawn({ let local_directory_path = local_directory_path.clone(); @@ -152,7 +148,7 @@ pub async fn start_event_loop( async move { while let Some((tx, callback)) = rcv_tx_event_rx.recv().await { //create new event with the Tx - let event: EventTx = tx.into(); + let event: TxEvent = tx.into(); //process RcvTx(EventTx) event let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 39bad833..d4b3fe34 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -139,7 +139,7 @@ impl From for File { } } -//describe a file generated by the VM +// Describe a file generated by the VM. #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct Vm(Hash); impl File { @@ -150,7 +150,7 @@ impl File { pub fn get_relatif_path(&self) -> PathBuf { let mut file_path = Path::new(&self.name); if file_path.is_absolute() { - file_path = file_path.strip_prefix("/").unwrap(); //unwrap tested in is_absolute + file_path = file_path.strip_prefix("/").unwrap(); // Unwrap tested in `is_absolute()`. } let mut path = PathBuf::from(self.extention.0.to_string()); @@ -191,10 +191,6 @@ pub async fn move_vmfile( #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] pub struct Image(Hash); impl File { - // pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { - // File::build(name, url, checksum, Image(tx_hash)) - // } - pub fn try_from_prg_meta_data(value: &transaction::ProgramMetadata) -> Self { File::build( value.name.clone(), diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index 9814bb24..c4f457f9 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -2,7 +2,6 @@ use super::file::{Download, File, Image, ProofVerif}; use super::signature::Signature; use super::{hash::Hash, program::ResourceRequest}; use crate::types::transaction; -use async_trait::async_trait; use eyre::Result; use libsecp256k1::{sign, verify, Message, PublicKey, SecretKey}; use num_bigint::BigInt; @@ -11,19 +10,6 @@ use sha3::{Digest, Sha3_256}; use std::{collections::HashSet, rc::Rc}; use thiserror::Error; -#[async_trait] -pub trait AclWhitelist: Send + Sync { - async fn contains(&self, key: &PublicKey) -> Result; -} - -pub struct AlwaysGrantAclWhitelist; -#[async_trait::async_trait] -impl AclWhitelist for AlwaysGrantAclWhitelist { - async fn contains(&self, _key: &PublicKey) -> Result { - Ok(true) - } -} - #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub enum TransactionTree { Root { @@ -263,29 +249,29 @@ pub enum TransactionError { // Tx are defined in 3 domains: Validation, Execution, Storage. // Currently the same definition is used but different type should be defined (TODO). // Only the validation type state are defined. -// TxCreate : identify a Tx that has just been created. -// TxReceive: Identify a Tx that has been received. Determine the received source. -// TxValdiated: Identify a Tx that has been validated. Pass all the validation process. +// Created : identify a Tx that has just been created. +// Received: Identify a Tx that has been received. Determine the received source. +// Validated: Identify a Tx that has been validated. Pass all the validation process. // The validation suppose the Tx has been propagated. Currently there's no notification during the propagation (TODO). #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct TxCreate; +pub struct Created; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub enum TxReceive { +pub enum Received { P2P, RPC, TXRESULT, } -impl TxReceive { +impl Received { fn is_from_tx_exec_result(&self) -> bool { - matches!(self, TxReceive::TXRESULT) + matches!(self, Received::TXRESULT) } } #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] -pub struct TxValdiated; +pub struct Validated; #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] pub struct Transaction { @@ -313,7 +299,7 @@ impl Transaction { } } -impl Default for Transaction { +impl Default for Transaction { fn default() -> Self { Self { author: PublicKey::from_secret_key(&SecretKey::default()), @@ -323,12 +309,12 @@ impl Default for Transaction { signature: Signature::default(), propagated: false, executed: false, - state: TxCreate, + state: Created, } } } -impl Default for Transaction { +impl Default for Transaction { fn default() -> Self { Self { author: PublicKey::from_secret_key(&SecretKey::default()), @@ -338,12 +324,12 @@ impl Default for Transaction { signature: Signature::default(), propagated: false, executed: false, - state: TxValdiated, + state: Validated, } } } -impl Transaction { +impl Transaction { pub fn new(payload: Payload, signing_key: &SecretKey) -> Self { let author = PublicKey::from_secret_key(signing_key); @@ -355,7 +341,7 @@ impl Transaction { signature: Signature::default(), propagated: false, executed: false, - state: TxCreate, + state: Created, }; tx.sign(signing_key); @@ -371,7 +357,7 @@ impl Transaction { self.signature = sig.into(); } - pub fn into_received(self, state: TxReceive) -> Transaction { + pub fn into_received(self, state: Received) -> Transaction { Transaction { author: self.author, hash: self.hash, @@ -385,7 +371,7 @@ impl Transaction { } } -impl Transaction { +impl Transaction { pub fn verify(&self) -> bool { let hash = self.compute_hash(); let msg: Message = hash.into(); @@ -483,7 +469,7 @@ mod tests { let sk = SecretKey::random(&mut StdRng::from_entropy()); let tx = Transaction::new(Payload::Empty, &sk); - let tx = tx.into_received(TxReceive::P2P); + let tx = tx.into_received(Received::P2P); assert!(tx.verify()); } @@ -497,7 +483,7 @@ mod tests { tx.nonce += 1; // Verify must return false. - let tx = tx.into_received(TxReceive::TXRESULT); + let tx = tx.into_received(Received::TXRESULT); assert!(!tx.verify()); } @@ -512,22 +498,22 @@ mod tests { }; let sk = SecretKey::random(&mut StdRng::from_entropy()); - let tx = Transaction:: { + let tx = Transaction:: { author: PublicKey::from_secret_key(&sk), payload: Payload::Run { workflow }, signature: Signature::default(), ..Default::default() }; - let tx = tx.into_received(TxReceive::RPC); + let tx = tx.into_received(Received::RPC); assert!(tx.validate().is_err()); } #[test] fn test_tx_validations_verifies_signature() { - let tx = Transaction::::default(); + let tx = Transaction::::default(); - let tx = tx.into_received(TxReceive::RPC); + let tx = tx.into_received(Received::RPC); assert!(tx.validate().is_err()); } } diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index 2f6d30e0..e92a5197 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -141,7 +141,7 @@ impl Provider for Qemu { // TODO: // - Builder to construct QEMU flags // - Handle GPUs - // - verify the file exist before lauching. Avoid to panic the node because Qemu break. + // - Verify that the file exists before booting the VM. Otherwise the node panics because the QEMU won't start. let img_file = Path::new(&self.config.data_directory) .join(IMAGES_DIR) diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index 0c0b25b7..d0b7038f 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -2,7 +2,7 @@ use crate::types::file::DbFile; use async_trait::async_trait; use eyre::Result; use gevulot_node::types::{ - transaction::{Payload, TxValdiated, Workflow, WorkflowStep}, + transaction::{Payload, Validated, Workflow, WorkflowStep}, Hash, Task, TaskKind, Transaction, }; use std::sync::Arc; @@ -30,7 +30,7 @@ pub enum WorkflowError { #[async_trait] pub trait TransactionStore: Sync + Send { - async fn find_transaction(&self, tx_hash: &Hash) -> Result>>; + async fn find_transaction(&self, tx_hash: &Hash) -> Result>>; } pub struct WorkflowEngine { @@ -42,7 +42,7 @@ impl WorkflowEngine { WorkflowEngine { tx_store } } - pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { + pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { let workflow = self.workflow_for_transaction(&cur_tx.hash).await?; match &cur_tx.payload { @@ -294,7 +294,7 @@ impl WorkflowEngine { #[cfg(test)] mod tests { - use gevulot_node::types::transaction::TxCreate; + use gevulot_node::types::transaction::Created; use std::collections::HashMap; use gevulot_node::types::{ @@ -307,11 +307,11 @@ mod tests { use super::*; pub struct TxStore { - pub txs: HashMap>, + pub txs: HashMap>, } impl TxStore { - pub fn new(txs: &[Transaction]) -> Self { + pub fn new(txs: &[Transaction]) -> Self { let mut store = TxStore { txs: HashMap::with_capacity(txs.len()), }; @@ -324,10 +324,7 @@ mod tests { #[async_trait] impl TransactionStore for TxStore { - async fn find_transaction( - &self, - tx_hash: &Hash, - ) -> Result>> { + async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { Ok(self.txs.get(tx_hash).cloned()) } } @@ -407,7 +404,7 @@ mod tests { assert!(task.is_ok()); } - fn into_validated(tx: Transaction) -> Transaction { + fn into_validated(tx: Transaction) -> Transaction { Transaction { author: tx.author, hash: tx.hash, @@ -416,11 +413,11 @@ mod tests { signature: tx.signature, propagated: tx.executed, executed: tx.executed, - state: TxValdiated, + state: Validated, } } - fn transaction_for_workflow_steps(steps: Vec) -> Transaction { + fn transaction_for_workflow_steps(steps: Vec) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); into_validated(Transaction::new( Payload::Run { @@ -430,7 +427,7 @@ mod tests { )) } - fn transaction_for_proof(parent: &Hash, program: &Hash) -> Transaction { + fn transaction_for_proof(parent: &Hash, program: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); into_validated(Transaction::new( Payload::Proof { @@ -443,7 +440,7 @@ mod tests { )) } - fn transaction_for_proofkey(parent: &Hash) -> Transaction { + fn transaction_for_proofkey(parent: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); into_validated(Transaction::new( Payload::ProofKey { @@ -454,7 +451,7 @@ mod tests { )) } - fn transaction_for_verification(parent: &Hash, program: &Hash) -> Transaction { + fn transaction_for_verification(parent: &Hash, program: &Hash) -> Transaction { let key = SecretKey::random(&mut StdRng::from_entropy()); into_validated(Transaction::new( Payload::Verification { diff --git a/crates/shim/proto/vm_service.proto b/crates/shim/proto/vm_service.proto index 60d99b01..d9497e4b 100644 --- a/crates/shim/proto/vm_service.proto +++ b/crates/shim/proto/vm_service.proto @@ -43,7 +43,7 @@ message GetFileRequest { message File { string path = 1; - bytes checksum = 3; + bytes checksum = 2; } message FileMetadata { diff --git a/crates/tests/e2e-tests/src/main.rs b/crates/tests/e2e-tests/src/main.rs index 5f7f2fd8..eaca0a12 100644 --- a/crates/tests/e2e-tests/src/main.rs +++ b/crates/tests/e2e-tests/src/main.rs @@ -1,4 +1,4 @@ -use gevulot_node::types::transaction::TxCreate; +use gevulot_node::types::transaction::Created; use std::{ net::SocketAddr, path::{Path, PathBuf}, @@ -109,7 +109,7 @@ async fn deploy_programs( signature: read_tx.signature, propagated: false, executed: false, - state: TxCreate, + state: Created, }; assert_eq!(tx, read_tx); From f422f1430a4f8b1a83d5239662be270da06bc03e Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 19 Feb 2024 10:46:38 +0100 Subject: [PATCH 28/43] add newline to be like the original file --- crates/node/migrations/20231128120351_transactions-table.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/migrations/20231128120351_transactions-table.sql b/crates/node/migrations/20231128120351_transactions-table.sql index 27f8cc5d..8ce61ec2 100644 --- a/crates/node/migrations/20231128120351_transactions-table.sql +++ b/crates/node/migrations/20231128120351_transactions-table.sql @@ -116,4 +116,4 @@ CREATE TABLE proof_key ( CONSTRAINT fk_transaction FOREIGN KEY (tx) REFERENCES transaction (hash) ON DELETE CASCADE -); \ No newline at end of file +); From a0216cc04e2fed74cc746e567b9ac88a76b4ddab Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 19 Feb 2024 15:24:29 +0100 Subject: [PATCH 29/43] change generated file name from uuid to checksum --- crates/node/src/networking/p2p/pea2pea.rs | 2 +- crates/node/src/scheduler/mod.rs | 21 ++++-------- .../node/src/txvalidation/download_manager.rs | 2 +- crates/node/src/txvalidation/mod.rs | 2 +- crates/node/src/types/file.rs | 32 ++++++++----------- crates/node/src/workflow/mod.rs | 3 +- 6 files changed, 24 insertions(+), 38 deletions(-) diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index b979cd25..618efc77 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -473,7 +473,7 @@ mod tests { (peer, tx_sender, txreceiver1) } - //TODO change by impl From when module declaration between main and lib are solved. + // TODO: Change to `impl From` form when module declaration between main and lib is solved. fn into_receive(tx: Transaction) -> Transaction { Transaction { author: tx.author, diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 463ec593..64b533fd 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -9,7 +9,12 @@ use crate::types::TaskState; use crate::vmm::vm_server::grpc; use crate::vmm::{vm_server::TaskManager, VMId}; use crate::workflow::{WorkflowEngine, WorkflowError}; +use crate::{ + mempool::Mempool, + types::{Hash, Task}, +}; use async_trait::async_trait; +use eyre::Result; use gevulot_node::types::transaction::Payload; use gevulot_node::types::{TaskKind, Transaction}; use libsecp256k1::SecretKey; @@ -17,26 +22,17 @@ pub use program_manager::ProgramManager; use rand::RngCore; pub use resource_manager::ResourceManager; use std::path::PathBuf; -use uuid::Uuid; - use std::time::Instant; use std::{ collections::{HashMap, VecDeque}, sync::Arc, time::Duration, }; - -use eyre::Result; use tokio::{ sync::{Mutex, RwLock}, time::sleep, }; -use crate::{ - mempool::Mempool, - types::{Hash, Task}, -}; - use self::program_manager::{ProgramError, ProgramHandle}; use self::resource_manager::ResourceError; @@ -422,7 +418,7 @@ impl TaskManager for Scheduler { ); } - //move and build output files of the Tx execution + // Handle tx execution's result files so that they are available as an input for next task if needed. let executed_files: Vec<(File, File)> = result .files .into_iter() @@ -432,11 +428,6 @@ impl TaskManager for Scheduler { file.checksum[..].into(), running_task.task.tx, ); - - let uuid = Uuid::new_v4(); - //format file_name to keep the current filename and the uuid. - //put uuid at the end so that the uuid is use to save the file. - let new_file_name = format!("{}/{}", file.path, uuid); let dest = File::::new( file.path, self.http_download_host.clone(), diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 308f66bc..0a6c00a4 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -25,7 +25,7 @@ pub async fn download_asset_file( http_client: &reqwest::Client, asset_file: File, ) -> Result<()> { - let local_relative_file_path = asset_file.get_relatif_path(); + let local_relative_file_path = asset_file.get_save_path(); tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); let mut resp = match tokio::time::timeout( tokio::time::Duration::from_secs(5), diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 8095e7d9..5da91968 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -55,7 +55,7 @@ pub struct RpcSender; pub struct P2pSender; pub struct TxResultSender; -//use to send a tx to the event process. +// `TxEventSender` holds the received transaction of a specific state together with an optional callback interface. #[derive(Debug, Clone)] pub struct TxEventSender { sender: UnboundedSender<(Transaction, Option)>, diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index d4b3fe34..036a2492 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -89,8 +89,11 @@ impl File { File::build(name, url, checksum, Download(tx_hash.to_string())) } - pub fn get_relatif_path(&self) -> PathBuf { - let file_name = Path::new(&self.name).file_name().unwrap(); + // Save Relative File path for downloaded files is / + pub fn get_save_path(&self) -> PathBuf { + let file_name = Path::new(&self.name) + .file_name() + .unwrap_or(std::ffi::OsStr::new("")); let mut path = PathBuf::from(&self.extention.0); path.push(file_name); path @@ -101,30 +104,21 @@ impl File { pub struct ProofVerif; impl File { pub fn new(path: String, http_download_host: String, checksum: Hash) -> Self { - let uuid = uuid::Uuid::new_v4(); - //format file_name to keep the current filename and the uuid. - //put uuid at the end so that the uuid is use to save the file. - let new_file_name = format!("{}/{}", path, uuid); - - File::build(new_file_name, http_download_host, checksum, ProofVerif) + File::build(path, http_download_host, checksum, ProofVerif) } pub fn into_download_file(self, tx_hash: Hash) -> File { - //get uuid from file name - let uuid = Path::new(&self.name).file_name().unwrap(); - File::::new( - self.name.to_string(), - format!("{}/{}/{}", self.url, tx_hash, uuid.to_str().unwrap()), - self.checksum.to_string().into(), - tx_hash, - ) + let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); + let url = format!("{}/{}", self.url, relative_path); + File::::new(relative_path, url, self.checksum.clone(), tx_hash) } + // Save Relative File path for ProofVerif files is // pub fn get_relatif_path(&self, tx_hash: Hash) -> PathBuf { - //get uuid from file name - let uuid = Path::new(&self.name).file_name().unwrap(); + let file_name = Path::new(&self.name).file_name().unwrap_or_default(); let mut path = PathBuf::from(tx_hash.to_string()); - path.push(uuid); + path.push(self.checksum.to_string()); + path.push(file_name); path } diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index d0b7038f..b3f675dc 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -245,7 +245,8 @@ impl WorkflowEngine { continue; } Payload::Verification { parent, .. } => { - //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db + //// XXX: If we return the parent tx, it gets re-executed and would generate + //// a duplicate key value violates unique constraint error in the db tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); tx_hash = parent; continue; From 02a13470127d189d9cb752dbb2449031dcb0ac52 Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 19 Feb 2024 17:25:59 +0100 Subject: [PATCH 30/43] Avoid to move download existing file --- .../node/src/txvalidation/download_manager.rs | 12 ++- crates/node/src/types/file.rs | 94 ++++++++++++++----- crates/node/src/types/transaction.rs | 1 + 3 files changed, 80 insertions(+), 27 deletions(-) diff --git a/crates/node/src/txvalidation/download_manager.rs b/crates/node/src/txvalidation/download_manager.rs index 0a6c00a4..cb86e50e 100644 --- a/crates/node/src/txvalidation/download_manager.rs +++ b/crates/node/src/txvalidation/download_manager.rs @@ -18,7 +18,7 @@ use tokio::net::TcpListener; use tokio::task::JoinHandle; use tokio_util::io::ReaderStream; -/// download_file downloads file from the given `url` and saves it to file in `local_directory_path` + / + `file`. +/// Downloads file from the given `url` and saves it to file in `local_directory_path` + / + `file path`. pub async fn download_asset_file( local_directory_path: &Path, http_peer_list: &[(SocketAddr, Option)], @@ -27,6 +27,16 @@ pub async fn download_asset_file( ) -> Result<()> { let local_relative_file_path = asset_file.get_save_path(); tracing::info!("download_file:{asset_file:?} local_directory_path:{local_directory_path:?} local_relative_file_path:{local_relative_file_path:?} http_peer_list:{http_peer_list:?}"); + + // Detect if the file already exist. If yes don't download. + if asset_file.exist(&local_relative_file_path).await { + tracing::trace!( + "download_asset_file: File already exist, skip download: {:#?}", + asset_file.get_save_path() + ); + return Ok(()); + } + let mut resp = match tokio::time::timeout( tokio::time::Duration::from_secs(5), http_client.get(asset_file.url).send(), diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 036a2492..591f1cd9 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -82,11 +82,24 @@ impl File { //File type #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] -pub struct Download(String); -//Asset file use download and install the file data locally. +pub struct Download((String, bool)); +// Asset file use download and install the file data locally. +// verify_exist: define if the exist verification do a real file system verification. +// Only file path using checksum should do file exist verification. impl File { - pub fn new(name: String, url: String, checksum: Hash, tx_hash: Hash) -> Self { - File::build(name, url, checksum, Download(tx_hash.to_string())) + pub fn new( + name: String, + url: String, + checksum: Hash, + tx_hash: Hash, + verify_exist: bool, + ) -> Self { + File::build( + name, + url, + checksum, + Download((tx_hash.to_string(), verify_exist)), + ) } // Save Relative File path for downloaded files is / @@ -94,10 +107,19 @@ impl File { let file_name = Path::new(&self.name) .file_name() .unwrap_or(std::ffi::OsStr::new("")); - let mut path = PathBuf::from(&self.extention.0); + let mut path = PathBuf::from(&self.extention.0 .0); path.push(file_name); path } + + pub async fn exist(&self, root_path: &Path) -> bool { + if self.extention.0 .1 { + let file_path = root_path.join(&root_path); + tokio::fs::try_exists(file_path).await.unwrap_or(false) + } else { + false + } + } } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] @@ -110,7 +132,7 @@ impl File { pub fn into_download_file(self, tx_hash: Hash) -> File { let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); let url = format!("{}/{}", self.url, relative_path); - File::::new(relative_path, url, self.checksum.clone(), tx_hash) + File::::new(relative_path, url, self.checksum.clone(), tx_hash, true) } // Save Relative File path for ProofVerif files is // @@ -125,6 +147,11 @@ impl File { pub fn vec_to_bytes(vec: &[File]) -> Result, bincode::Error> { bincode::serialize(vec) } + + pub async fn exist(&self, root_path: &Path) -> bool { + let file_path = root_path.join(&root_path); + tokio::fs::try_exists(file_path).await.unwrap_or(false) + } } impl From for File { @@ -151,6 +178,11 @@ impl File { path.push(file_path); path } + + pub async fn remove_file(&self, base_path: &Path) -> std::io::Result<()> { + let src_file_path = base_path.join(self.get_relatif_path()); + tokio::fs::remove_file(src_file_path).await + } } pub async fn move_vmfile( @@ -159,27 +191,37 @@ pub async fn move_vmfile( base_path: &Path, proofverif_tx_hash: Hash, ) -> Result<()> { - let src_file_path = base_path.to_path_buf().join(source.get_relatif_path()); - let dst_file_path = base_path - .to_path_buf() - .join(dest.get_relatif_path(proofverif_tx_hash)); - - tracing::debug!( - "moving file from {:#?} to {:#?}", - src_file_path, - dst_file_path - ); - - // Ensure any necessary subdirectories exists. - if let Some(parent) = dst_file_path.parent() { - tokio::fs::create_dir_all(parent) + // If the dest file already exist don't copy it. + // Remove it from the VM. + if dest.exist(base_path).await { + tracing::debug!( + "move_vmfile: dest file already exist. Rmove VM file:{:#?}", + source.get_relatif_path() + ); + source.remove_file(base_path).await.map_err(|e| e.into()) + } else { + let src_file_path = base_path.to_path_buf().join(source.get_relatif_path()); + let dst_file_path = base_path + .to_path_buf() + .join(dest.get_relatif_path(proofverif_tx_hash)); + + tracing::debug!( + "move_vmfile: moving file from {:#?} to {:#?}", + src_file_path, + dst_file_path + ); + + // Ensure any necessary subdirectories exists. + if let Some(parent) = dst_file_path.parent() { + tokio::fs::create_dir_all(parent) + .await + .expect("task file mkdir"); + } + + tokio::fs::rename(src_file_path, dst_file_path) .await - .expect("task file mkdir"); + .map_err(|e| e.into()) } - - tokio::fs::rename(src_file_path, dst_file_path) - .await - .map_err(|e| e.into()) } #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq)] @@ -204,7 +246,7 @@ impl From> for File { file.name, file.url, file.checksum, - Download(extention.to_str().unwrap().to_string()), + Download((extention.to_str().unwrap().to_string(), false)), ) } } diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index c4f457f9..b5414d32 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -435,6 +435,7 @@ impl Transaction { file_url.clone(), checksum.to_string().into(), self.hash, + false, )) }) .collect() From d45fb19d87353b23a1c54bc35d614eb59a2733ac Mon Sep 17 00:00:00 2001 From: musitdev Date: Mon, 19 Feb 2024 17:43:19 +0100 Subject: [PATCH 31/43] correct clippy --- crates/node/src/types/file.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index 591f1cd9..cab7f20f 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -114,7 +114,7 @@ impl File { pub async fn exist(&self, root_path: &Path) -> bool { if self.extention.0 .1 { - let file_path = root_path.join(&root_path); + let file_path = root_path.join(root_path); tokio::fs::try_exists(file_path).await.unwrap_or(false) } else { false @@ -132,7 +132,7 @@ impl File { pub fn into_download_file(self, tx_hash: Hash) -> File { let relative_path = self.get_relatif_path(tx_hash).to_str().unwrap().to_string(); let url = format!("{}/{}", self.url, relative_path); - File::::new(relative_path, url, self.checksum.clone(), tx_hash, true) + File::::new(relative_path, url, self.checksum, tx_hash, true) } // Save Relative File path for ProofVerif files is // @@ -149,7 +149,7 @@ impl File { } pub async fn exist(&self, root_path: &Path) -> bool { - let file_path = root_path.join(&root_path); + let file_path = root_path.join(root_path); tokio::fs::try_exists(file_path).await.unwrap_or(false) } } @@ -195,7 +195,7 @@ pub async fn move_vmfile( // Remove it from the VM. if dest.exist(base_path).await { tracing::debug!( - "move_vmfile: dest file already exist. Rmove VM file:{:#?}", + "move_vmfile: dest file already exist. Remove VM file:{:#?}", source.get_relatif_path() ); source.remove_file(base_path).await.map_err(|e| e.into()) From d68428bcd996a30049cf69f99aa5744201dae85c Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 20 Feb 2024 09:31:08 +0100 Subject: [PATCH 32/43] remove asset manager call to test --- crates/node/src/main.rs | 2 +- crates/node/src/storage/database/postgres.rs | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 1a45386f..43dc9cac 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -142,7 +142,7 @@ impl mempool::Storage for storage::Database { async fn set(&self, tx: &Transaction) -> Result<()> { let tx_hash = tx.hash; self.add_transaction(tx).await?; - self.add_asset(&tx_hash).await?; + // self.add_asset(&tx_hash).await?; self.mark_asset_complete(&tx_hash).await } diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index bd920df4..75310689 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -224,15 +224,15 @@ impl Database { Ok(()) } - pub async fn add_asset(&self, tx_hash: &Hash) -> Result<()> { - sqlx::query!( - "INSERT INTO assets ( tx ) VALUES ( $1 ) RETURNING *", - tx_hash.to_string(), - ) - .fetch_one(&self.pool) - .await?; - Ok(()) - } + // pub async fn add_asset(&self, tx_hash: &Hash) -> Result<()> { + // sqlx::query!( + // "INSERT INTO assets ( tx ) VALUES ( $1 ) RETURNING *", + // tx_hash.to_string(), + // ) + // .fetch_one(&self.pool) + // .await?; + // Ok(()) + // } pub async fn has_assets_loaded(&self, tx_hash: &Hash) -> Result { let res: Option = From d44e527e7eabcc1cc161e4ffdcee4d0c26ce1f2c Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 20 Feb 2024 10:58:11 +0100 Subject: [PATCH 33/43] correct some path issue --- crates/node/src/types/file.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/crates/node/src/types/file.rs b/crates/node/src/types/file.rs index cab7f20f..684db250 100644 --- a/crates/node/src/types/file.rs +++ b/crates/node/src/types/file.rs @@ -114,7 +114,7 @@ impl File { pub async fn exist(&self, root_path: &Path) -> bool { if self.extention.0 .1 { - let file_path = root_path.join(root_path); + let file_path = root_path.join(self.get_save_path()); tokio::fs::try_exists(file_path).await.unwrap_or(false) } else { false @@ -148,8 +148,8 @@ impl File { bincode::serialize(vec) } - pub async fn exist(&self, root_path: &Path) -> bool { - let file_path = root_path.join(root_path); + pub async fn exist(&self, root_path: &Path, tx_hash: Hash) -> bool { + let file_path = root_path.join(self.get_relatif_path(tx_hash)); tokio::fs::try_exists(file_path).await.unwrap_or(false) } } @@ -193,9 +193,10 @@ pub async fn move_vmfile( ) -> Result<()> { // If the dest file already exist don't copy it. // Remove it from the VM. - if dest.exist(base_path).await { + if dest.exist(base_path, proofverif_tx_hash).await { tracing::debug!( - "move_vmfile: dest file already exist. Remove VM file:{:#?}", + "move_vmfile: dest file already exist:{:#?}. Remove VM file:{:#?}", + dest.get_relatif_path(proofverif_tx_hash), source.get_relatif_path() ); source.remove_file(base_path).await.map_err(|e| e.into()) From ff72aedf11139043dd7639a9cd35cb1a2c7ad60e Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 20 Feb 2024 11:39:39 +0100 Subject: [PATCH 34/43] add verify tx execution in the workflow. --- .gitignore | 1 + ...6c71e6f6805787aae1dea47cb248d07689498.json | 34 ------------------- crates/node/src/main.rs | 3 ++ crates/node/src/storage/database/postgres.rs | 1 + crates/node/src/workflow/mod.rs | 13 +++++++ 5 files changed, 18 insertions(+), 34 deletions(-) delete mode 100644 crates/node/.sqlx/query-151f420a22124e7c5f7d6a868f06c71e6f6805787aae1dea47cb248d07689498.json diff --git a/.gitignore b/.gitignore index 03a153a5..449e842b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ .DS_Store .idea *.pki +**/.sqlx crates/tests/e2e-tests/files crates/tests/e2e-tests/prover crates/tests/e2e-tests/verifier diff --git a/crates/node/.sqlx/query-151f420a22124e7c5f7d6a868f06c71e6f6805787aae1dea47cb248d07689498.json b/crates/node/.sqlx/query-151f420a22124e7c5f7d6a868f06c71e6f6805787aae1dea47cb248d07689498.json deleted file mode 100644 index 758221cf..00000000 --- a/crates/node/.sqlx/query-151f420a22124e7c5f7d6a868f06c71e6f6805787aae1dea47cb248d07689498.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO assets ( tx ) VALUES ( $1 ) RETURNING *", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tx", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "created", - "type_info": "Timestamp" - }, - { - "ordinal": 2, - "name": "completed", - "type_info": "Timestamp" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "151f420a22124e7c5f7d6a868f06c71e6f6805787aae1dea47cb248d07689498" -} diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index 43dc9cac..3c1e57ac 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -163,6 +163,9 @@ impl workflow::TransactionStore for storage::Database { async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { self.find_transaction(tx_hash).await } + async fn mark_tx_executed(&self, tx_hash: &Hash) -> Result<()> { + self.mark_tx_executed(tx_hash).await + } } async fn run(config: Arc) -> Result<()> { diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index 75310689..c6d80616 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -622,6 +622,7 @@ impl Database { verification, files, } => { + tracing::trace!("Postgres add_transaction tx:{}", tx.hash.to_string()); sqlx::query( "INSERT INTO verification ( tx, parent, verifier, verification ) VALUES ( $1, $2, $3, $4 ) ON CONFLICT (tx) DO NOTHING", ) diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index b3f675dc..a2817a4c 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -31,6 +31,7 @@ pub enum WorkflowError { #[async_trait] pub trait TransactionStore: Sync + Send { async fn find_transaction(&self, tx_hash: &Hash) -> Result>>; + async fn mark_tx_executed(&self, tx_hash: &Hash) -> Result<()>; } pub struct WorkflowEngine { @@ -149,6 +150,14 @@ impl WorkflowEngine { Err(WorkflowError::IncompatibleTransaction(proof_tx.hash.to_string()).into()) } } + Payload::Verification { .. } => { + // Execute the verify tx by setting to executed. + // Ideally it's not the right place to execute a Tx + // but as the execution is nothing, it's more convenient. + tracing::debug!("Mark as executed Payload::Verification tx {}", &cur_tx.hash); + self.tx_store.mark_tx_executed(&cur_tx.hash).await?; + Ok(None) + } _ => Err(WorkflowError::IncompatibleTransaction( "unsupported payload type".to_string(), ) @@ -328,6 +337,10 @@ mod tests { async fn find_transaction(&self, tx_hash: &Hash) -> Result>> { Ok(self.txs.get(tx_hash).cloned()) } + async fn mark_tx_executed(&self, tx_hash: &Hash) -> Result<()> { + // Do nothing because the txs map can't be modified behind a &self. + Ok(()) + } } #[tokio::test] From ee8a376f1999b7b2c742ada771db0e750f998449 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 20 Feb 2024 17:46:11 +0100 Subject: [PATCH 35/43] add mark deploy tx as executed --- crates/node/src/vmm/vm_server.rs | 19 ++++++++----------- crates/node/src/workflow/mod.rs | 29 ++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index b289c6ad..a6937f99 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -106,17 +106,14 @@ impl VmService for VMServer { }; let reply = match self.task_source.get_task(program, vm_id).await { - Some(task) => { - tracing::info!("task has {} files", task.files.len()); - grpc::TaskResponse { - result: Some(grpc::task_response::Result::Task(grpc::Task { - id: task.id.to_string(), - name: task.name.to_string(), - args: task.args, - files: task.files, - })), - } - } + Some(task) => grpc::TaskResponse { + result: Some(grpc::task_response::Result::Task(grpc::Task { + id: task.id.to_string(), + name: task.name.to_string(), + args: task.args, + files: task.files, + })), + }, None => grpc::TaskResponse { result: Some(grpc::task_response::Result::Error( grpc::TaskError::Unavailable.into(), diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index a2817a4c..b370d989 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -44,7 +44,7 @@ impl WorkflowEngine { } pub async fn next_task(&self, cur_tx: &Transaction) -> Result> { - let workflow = self.workflow_for_transaction(&cur_tx.hash).await?; + let opt_workflow = self.workflow_for_transaction(&cur_tx.hash).await?; match &cur_tx.payload { Payload::Run { workflow } => { @@ -70,6 +70,13 @@ impl WorkflowEngine { .. } => { tracing::debug!("creating next task from Proof tx {}", &cur_tx.hash); + let Some(workflow) = opt_workflow else { + return Err(WorkflowError::WorkflowTransactionMissing(format!( + "Proof tx with no workflow {}", + cur_tx.hash.clone(), + )) + .into()); + }; match workflow.steps.iter().position(|s| s.program == *prover) { Some(proof_step_idx) => { @@ -121,6 +128,13 @@ impl WorkflowEngine { .. } = proof_tx.payload { + let Some(workflow) = opt_workflow else { + return Err(WorkflowError::WorkflowTransactionMissing(format!( + "Proof tx with no workflow {}", + cur_tx.hash.clone(), + )) + .into()); + }; match workflow.steps.iter().position(|s| s.program == prover) { Some(proof_step_idx) => { if workflow.steps.len() <= proof_step_idx { @@ -158,6 +172,14 @@ impl WorkflowEngine { self.tx_store.mark_tx_executed(&cur_tx.hash).await?; Ok(None) } + Payload::Deploy { .. } => { + // Execute the Deploy tx by setting to executed. + // Ideally it's not the right place to execute a Tx + // but as the execution is only a move of file that has been done, it's more convenient. + tracing::debug!("Mark as executed Payload::Deploy tx {}", &cur_tx.hash); + self.tx_store.mark_tx_executed(&cur_tx.hash).await?; + Ok(None) + } _ => Err(WorkflowError::IncompatibleTransaction( "unsupported payload type".to_string(), ) @@ -223,7 +245,7 @@ impl WorkflowEngine { } } - async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result { + async fn workflow_for_transaction(&self, tx_hash: &Hash) -> Result> { let mut tx_hash = *tx_hash; tracing::debug!("finding workflow for transaction {}", tx_hash); @@ -240,7 +262,7 @@ impl WorkflowEngine { match tx.unwrap().payload { Payload::Run { workflow } => { tracing::debug!("workflow found for transaction {}", tx_hash); - return Ok(workflow); + return Ok(Some(workflow)); } Payload::Proof { parent, .. } => { //if we return the parent Tx it's reexecuted an generate a duplicate key value violates unique constraint error in the db @@ -260,6 +282,7 @@ impl WorkflowEngine { tx_hash = parent; continue; } + Payload::Deploy { .. } => return Ok(None), payload => { tracing::debug!( "failed to find workflow for transaction {}: incompatible transaction :{payload:?}", From 6d563e0e9b9282fa58fc4c61e98ddf35f80f1ef5 Mon Sep 17 00:00:00 2001 From: musitdev Date: Tue, 20 Feb 2024 19:08:46 +0100 Subject: [PATCH 36/43] remove the dbg! that panic --- crates/node/src/scheduler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 64b533fd..eaaabe54 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -392,7 +392,7 @@ impl TaskManager for Scheduler { vm_id: Arc, result: grpc::task_result_request::Result, ) -> bool { - dbg!(&result); + tracing::debug!("submit_result result:{result:?}"); let grpc::task_result_request::Result::Task(result) = result else { todo!("task failed; handle it correctly") From 29a99d8e606d1b49151f338385f7aef229836a40 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 09:41:57 +0100 Subject: [PATCH 37/43] change verif tx workflow management and change logs --- crates/node/src/scheduler/mod.rs | 2 +- crates/node/src/workflow/mod.rs | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index eaaabe54..5026f0ef 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -392,7 +392,7 @@ impl TaskManager for Scheduler { vm_id: Arc, result: grpc::task_result_request::Result, ) -> bool { - tracing::debug!("submit_result result:{result:?}"); + tracing::debug!("submit_result result:{result:#?}"); let grpc::task_result_request::Result::Task(result) = result else { todo!("task failed; handle it correctly") diff --git a/crates/node/src/workflow/mod.rs b/crates/node/src/workflow/mod.rs index b370d989..877faaf2 100644 --- a/crates/node/src/workflow/mod.rs +++ b/crates/node/src/workflow/mod.rs @@ -276,11 +276,14 @@ impl WorkflowEngine { continue; } Payload::Verification { parent, .. } => { - //// XXX: If we return the parent tx, it gets re-executed and would generate - //// a duplicate key value violates unique constraint error in the db - tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); - tx_hash = parent; - continue; + // //// XXX: If we return the parent tx, it gets re-executed and would generate + // //// a duplicate key value violates unique constraint error in the db + // tracing::debug!("finding workflow from parent {} of {}", &parent, tx_hash); + // tx_hash = parent; + // continue; + + //no workflow for Verif Tx + return Ok(None); } Payload::Deploy { .. } => return Ok(None), payload => { From 8766908efffb8297d1af24c41ed9e3c9bef44dbf Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 12:26:57 +0100 Subject: [PATCH 38/43] add some logs --- crates/node/src/networking/p2p/pea2pea.rs | 3 ++- crates/node/src/txvalidation/event.rs | 12 ++++++++++-- crates/node/src/types/transaction.rs | 24 +++++++++++++++++++++++ crates/node/src/vmm/qemu.rs | 6 +++++- crates/node/src/vmm/vm_server.rs | 6 ++++++ 5 files changed, 47 insertions(+), 4 deletions(-) diff --git a/crates/node/src/networking/p2p/pea2pea.rs b/crates/node/src/networking/p2p/pea2pea.rs index 618efc77..f34d557c 100644 --- a/crates/node/src/networking/p2p/pea2pea.rs +++ b/crates/node/src/networking/p2p/pea2pea.rs @@ -370,8 +370,9 @@ impl Reading for P2P { Ok(protocol::Message::V0(msg)) => match msg { protocol::MessageV0::Transaction(tx) => { tracing::debug!( - "received transaction {} author:{}", + "received transaction {}:{} author:{}", tx.hash, + tx.payload, hex::encode(tx.author.serialize()) ); let tx: Transaction = Transaction { diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index b865fc62..cfeb8878 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -166,7 +166,11 @@ impl TxEvent { executed: self.tx.executed, state: Validated, }; - tracing::info!("Tx validation propagate tx:{}", tx.hash.to_string()); + tracing::info!( + "Tx validation propagate tx:{} payload:{}", + tx.hash.to_string(), + tx.payload + ); p2p_sender.send(tx).map_err(|err| Box::new(err).into()) } } @@ -184,7 +188,11 @@ impl TxEvent { executed: self.tx.executed, state: Validated, }; - tracing::info!("Tx validation save tx:{}", tx.hash.to_string()); + tracing::info!( + "Tx validation save tx:{} payload:{}", + tx.hash.to_string(), + tx.payload + ); mempool .add(tx) .map_err(|err| EventProcessError::SaveTxError(format!("{err}"))) diff --git a/crates/node/src/types/transaction.rs b/crates/node/src/types/transaction.rs index b5414d32..7dff9e18 100644 --- a/crates/node/src/types/transaction.rs +++ b/crates/node/src/types/transaction.rs @@ -174,6 +174,24 @@ pub enum Payload { }, } +impl std::fmt::Display for Payload { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let payload = match self { + Payload::Empty => "Empty", + Payload::Transfer { .. } => "Transfer", + Payload::Stake { .. } => "Stake", + Payload::Unstake { .. } => "Unstake", + Payload::Deploy { .. } => "Deploy", + Payload::Run { .. } => "Run", + Payload::Proof { .. } => "Proof", + Payload::ProofKey { .. } => "ProofKey", + Payload::Verification { .. } => "Verification", + Payload::Cancel { .. } => "Cancel", + }; + write!(f, "({})", payload) + } +} + impl Payload { pub fn serialize_into(&self, buf: &mut Vec) { match self { @@ -346,6 +364,12 @@ impl Transaction { tx.sign(signing_key); + tracing::debug!( + "Transaction::new tx:{} payload:{}", + tx.hash.to_string(), + tx.payload + ); + tx } diff --git a/crates/node/src/vmm/qemu.rs b/crates/node/src/vmm/qemu.rs index e92a5197..f84c26b1 100644 --- a/crates/node/src/vmm/qemu.rs +++ b/crates/node/src/vmm/qemu.rs @@ -266,7 +266,11 @@ impl Provider for Qemu { cmd.stderr(Stdio::from(stderr)); } - tracing::info!("starting QEMU. args:\n{:#?}\n", cmd.get_args()); + tracing::info!( + "Program:{} starting QEMU. args:\n{:#?}\n", + program.hash.to_string(), + cmd.get_args(), + ); qemu_vm_handle.child = Some(cmd.spawn().expect("failed to start VM")); diff --git a/crates/node/src/vmm/vm_server.rs b/crates/node/src/vmm/vm_server.rs index a6937f99..77839f5b 100644 --- a/crates/node/src/vmm/vm_server.rs +++ b/crates/node/src/vmm/vm_server.rs @@ -120,6 +120,7 @@ impl VmService for VMServer { )), }, }; + tracing::trace!("VMServer get_task reply: {:?}", reply); Ok(Response::new(reply)) } @@ -302,6 +303,11 @@ impl VmService for VMServer { } }; + tracing::trace!( + "VMServer submit_result program:{}, vm_id:{vm_id}", + program.to_string() + ); + let result = request.into_inner().result; if let Some(result) = result { From 65b0b979d6948f702fb0d1f67ff470cbcbd3f052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tuomas=20M=C3=A4kinen?= <1947505+tuommaki@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:48:33 +0200 Subject: [PATCH 39/43] Terminate VM on get_task() when already assigned with Task (#96) Two major changes in this commit: - Terminate the VM if it queries for a new task when it already has a task assigned (in running_tasks). - Fix the `task_queue` operation: `front()` -> `pop_front()`. - When pulling a task from the queue, it must be removed from it to prevent re-execution. This mistake happened to accidentally work because the VM was stopped after successful task execution (so it didn't ask for a new task again). --- crates/node/src/scheduler/mod.rs | 45 +++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index 5026f0ef..e81676b0 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -353,8 +353,47 @@ impl TaskManager for Scheduler { vm_id ); - if let Some(task_queue) = self.task_queue.lock().await.get(&program) { - if let Some((task, scheduled)) = task_queue.front() { + // Ensure that the VM requesting for a task is not already executing one! + let mut running_tasks = self.running_tasks.lock().await; + if let Some(idx) = running_tasks.iter().position(|e| e.vm_id.eq(vm_id.clone())) { + // A VM that is already running a task, requests for a new one. + // Mark the existing task as executed and stop the VM. + let running_task = running_tasks.swap_remove(idx); + tracing::info!( + "task {} has been running {}sec but found to be orphaned. marking it as executed.", + running_task.task.id, + running_task.task_started.elapsed().as_secs() + ); + + if let Err(err) = self.database.mark_tx_executed(&running_task.task.tx).await { + tracing::error!( + "failed to update transaction.executed => true - tx.hash: {} error:{err}", + &running_task.task.tx + ); + } + + tracing::debug!("terminating VM {} running program {}", vm_id, program); + + let mut running_vms = self.running_vms.lock().await; + let idx = running_vms.iter().position(|e| e.vm_id().eq(vm_id.clone())); + if idx.is_some() { + let program_handle = running_vms.remove(idx.unwrap()); + if let Err(err) = self + .program_manager + .lock() + .await + .stop_program(program_handle) + .await + { + tracing::error!("failed to stop program {}: {}", program, err); + } + } + + return None; + } + + if let Some(task_queue) = self.task_queue.lock().await.get_mut(&program) { + if let Some((task, scheduled)) = task_queue.pop_front() { tracing::debug!( "task {} found for program {} running in vm_id {}", task.id, @@ -370,7 +409,7 @@ impl TaskManager for Scheduler { self.running_tasks.lock().await.push(RunningTask { task: task.clone(), vm_id: vm_id.clone(), - task_scheduled: *scheduled, + task_scheduled: scheduled, task_started: Instant::now(), }); From e4481f60ba683f4381d0063b437d484337373ef7 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 16:03:29 +0100 Subject: [PATCH 40/43] revert .gitignore change --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index 449e842b..03a153a5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ .DS_Store .idea *.pki -**/.sqlx crates/tests/e2e-tests/files crates/tests/e2e-tests/prover crates/tests/e2e-tests/verifier From 80998df3fa743a484e2cdda909c09f0bb14e07b9 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 16:41:18 +0100 Subject: [PATCH 41/43] correct mutex lock issue --- crates/node/src/scheduler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index e81676b0..b2f03820 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -406,7 +406,7 @@ impl TaskManager for Scheduler { scheduled.elapsed().as_millis() ); - self.running_tasks.lock().await.push(RunningTask { + running_tasks.push(RunningTask { task: task.clone(), vm_id: vm_id.clone(), task_scheduled: scheduled, From 3aaab1fc5aaf76f051e818000731bb3819193210 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 18:15:03 +0100 Subject: [PATCH 42/43] add some logs --- crates/node/src/txvalidation/event.rs | 1 + crates/node/src/txvalidation/mod.rs | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index cfeb8878..af82d386 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -127,6 +127,7 @@ impl TxEvent { let futures: Vec<_> = asset_file_list .into_iter() .map(|asset_file| { + tracing::trace!("txvalidation process_event download for file :{asset_file:?}",); download_manager::download_asset_file( local_directory_path, &http_peer_list, diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index 5da91968..dff58376 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -153,6 +153,8 @@ pub async fn spawn_event_loop( //process RcvTx(EventTx) event let http_peer_list = convert_peer_list_to_vec(&http_peer_list).await; + tracing::trace!("txvalidation receive event:{}", event.tx.hash.to_string()); + //process the receive event tokio::spawn({ let p2p_sender = p2p_sender.clone(); @@ -163,17 +165,33 @@ pub async fn spawn_event_loop( let res = event .process_event(acl_whitelist.as_ref()) .and_then(|download_event| { + tracing::trace!( + "txvalidation before download:{}", + download_event.tx.hash.to_string() + ); download_event.process_event(&local_directory_path, http_peer_list) }) .and_then(|(new_tx, propagate_tx)| async move { if let Some(propagate_tx) = propagate_tx { + tracing::trace!( + "txvalidation before propagate:{}", + propagate_tx.tx.hash.to_string() + ); propagate_tx.process_event(&p2p_sender).await?; } + tracing::trace!( + "txvalidation before new:{}", + new_tx.tx.hash.to_string() + ); new_tx.process_event(&mut *(mempool.write().await)).await?; Ok(()) }) .await; + //log the error if any error is return + if let Err(ref err) = res { + tracing::error!("An error occurs during Tx validation: {err}",); + } //send the execution result back if needed. if let Some(callback) = callback { //forget the result because if the RPC connection is closed the send can fail. From 37d7eb0c2b524fb6198c588dbbb0f589d099acb4 Mon Sep 17 00:00:00 2001 From: musitdev Date: Wed, 21 Feb 2024 20:55:32 +0100 Subject: [PATCH 43/43] remove asset management and some logs --- .../20240221184329_remove_asset_table.sql | 3 ++ crates/node/src/main.rs | 4 +- crates/node/src/scheduler/mod.rs | 14 ------- crates/node/src/storage/database/postgres.rs | 39 ------------------- crates/node/src/txvalidation/event.rs | 4 +- crates/node/src/txvalidation/mod.rs | 12 ------ 6 files changed, 5 insertions(+), 71 deletions(-) create mode 100644 crates/node/migrations/20240221184329_remove_asset_table.sql diff --git a/crates/node/migrations/20240221184329_remove_asset_table.sql b/crates/node/migrations/20240221184329_remove_asset_table.sql new file mode 100644 index 00000000..dcf53aba --- /dev/null +++ b/crates/node/migrations/20240221184329_remove_asset_table.sql @@ -0,0 +1,3 @@ +-- Remove asset table + +DROP TABLE IF EXISTS assets; diff --git a/crates/node/src/main.rs b/crates/node/src/main.rs index b259b1cf..0ce530f4 100644 --- a/crates/node/src/main.rs +++ b/crates/node/src/main.rs @@ -154,9 +154,7 @@ impl mempool::Storage for storage::Database { async fn set(&self, tx: &Transaction) -> Result<()> { let tx_hash = tx.hash; - self.add_transaction(tx).await?; - // self.add_asset(&tx_hash).await?; - self.mark_asset_complete(&tx_hash).await + self.add_transaction(tx).await } async fn fill_deque( diff --git a/crates/node/src/scheduler/mod.rs b/crates/node/src/scheduler/mod.rs index b2f03820..7df78d88 100644 --- a/crates/node/src/scheduler/mod.rs +++ b/crates/node/src/scheduler/mod.rs @@ -365,13 +365,6 @@ impl TaskManager for Scheduler { running_task.task_started.elapsed().as_secs() ); - if let Err(err) = self.database.mark_tx_executed(&running_task.task.tx).await { - tracing::error!( - "failed to update transaction.executed => true - tx.hash: {} error:{err}", - &running_task.task.tx - ); - } - tracing::debug!("terminating VM {} running program {}", vm_id, program); let mut running_vms = self.running_vms.lock().await; @@ -450,13 +443,6 @@ impl TaskManager for Scheduler { running_task.task_started.elapsed().as_secs() ); - if let Err(err) = self.database.mark_tx_executed(&running_task.task.tx).await { - tracing::error!( - "failed to update transaction.executed => true - tx.hash: {} error:{err}", - &running_task.task.tx - ); - } - // Handle tx execution's result files so that they are available as an input for next task if needed. let executed_files: Vec<(File, File)> = result .files diff --git a/crates/node/src/storage/database/postgres.rs b/crates/node/src/storage/database/postgres.rs index c6d80616..a0485351 100644 --- a/crates/node/src/storage/database/postgres.rs +++ b/crates/node/src/storage/database/postgres.rs @@ -224,45 +224,6 @@ impl Database { Ok(()) } - // pub async fn add_asset(&self, tx_hash: &Hash) -> Result<()> { - // sqlx::query!( - // "INSERT INTO assets ( tx ) VALUES ( $1 ) RETURNING *", - // tx_hash.to_string(), - // ) - // .fetch_one(&self.pool) - // .await?; - // Ok(()) - // } - - pub async fn has_assets_loaded(&self, tx_hash: &Hash) -> Result { - let res: Option = - sqlx::query("SELECT 1 FROM assets WHERE completed IS NOT NULL AND tx = $1") - .bind(tx_hash) - .map(|row: sqlx::postgres::PgRow| row.get(0)) - .fetch_optional(&self.pool) - .await?; - - Ok(res.is_some()) - } - - pub async fn get_incomplete_assets(&self) -> Result> { - let assets = - sqlx::query("SELECT tx FROM assets WHERE completed IS NULL ORDER BY created ASC") - .map(|row: sqlx::postgres::PgRow| row.get(0)) - .fetch_all(&self.pool) - .await?; - - Ok(assets) - } - - pub async fn mark_asset_complete(&self, tx_hash: &Hash) -> Result<()> { - sqlx::query("UPDATE assets SET completed = NOW() WHERE tx = $1") - .bind(&tx_hash.to_string()) - .execute(&self.pool) - .await?; - Ok(()) - } - // NOTE: There are plenty of opportunities for optimizations in following // transaction related operations. They are implemented naively on purpose // for now to maintain initial flexibility in development. Later on, these diff --git a/crates/node/src/txvalidation/event.rs b/crates/node/src/txvalidation/event.rs index af82d386..a0cdcac3 100644 --- a/crates/node/src/txvalidation/event.rs +++ b/crates/node/src/txvalidation/event.rs @@ -127,7 +127,6 @@ impl TxEvent { let futures: Vec<_> = asset_file_list .into_iter() .map(|asset_file| { - tracing::trace!("txvalidation process_event download for file :{asset_file:?}",); download_manager::download_asset_file( local_directory_path, &http_peer_list, @@ -141,8 +140,7 @@ impl TxEvent { .into_iter() .collect::, _>>() .map_err(|err| { - tracing::error!("Error during Tx file download:{err}"); - EventProcessError::DownloadAssetError(format!("Exwecution error:{err}")) + EventProcessError::DownloadAssetError(format!("Execution error:{err}")) })?; let newtx: TxEvent = self.clone().into(); let propagate: Option> = self.into(); diff --git a/crates/node/src/txvalidation/mod.rs b/crates/node/src/txvalidation/mod.rs index dff58376..c3bb117c 100644 --- a/crates/node/src/txvalidation/mod.rs +++ b/crates/node/src/txvalidation/mod.rs @@ -165,24 +165,12 @@ pub async fn spawn_event_loop( let res = event .process_event(acl_whitelist.as_ref()) .and_then(|download_event| { - tracing::trace!( - "txvalidation before download:{}", - download_event.tx.hash.to_string() - ); download_event.process_event(&local_directory_path, http_peer_list) }) .and_then(|(new_tx, propagate_tx)| async move { if let Some(propagate_tx) = propagate_tx { - tracing::trace!( - "txvalidation before propagate:{}", - propagate_tx.tx.hash.to_string() - ); propagate_tx.process_event(&p2p_sender).await?; } - tracing::trace!( - "txvalidation before new:{}", - new_tx.tx.hash.to_string() - ); new_tx.process_event(&mut *(mempool.write().await)).await?; Ok(())