Change materialize to use the parquet encoders and df_loaders

db4-storage/src/pages/mod.rs

@@ -176,7 +176,7 @@ impl<
             node_meta.get_or_create_node_type_id(node_type);
         }
 
-        let t_len = edge_storage.t_len();
+        let t_len = edge_storage.t_len() + node_storage.t_len();
 
         Ok(Self {
             nodes: node_storage,

db4-storage/src/pages/node_store.rs

@@ -170,6 +170,10 @@ impl<NS: NodeSegmentOps<Extension = EXT>, EXT: PersistenceStrategy<NS = NS>>
         self.segments.count()
     }
 
+    pub fn t_len(&self) -> usize {
+        self.segments.iter().map(|(_, page)| page.t_len()).sum()
+    }
+
     // pub fn segments(&self) -> &boxcar::Vec<Arc<NS>> {
     //     &self.segments
     // }

python/Cargo.toml

@@ -39,5 +39,5 @@ proto = ["raphtory/proto"]
 [build-dependencies]
 pyo3-build-config = { workspace = true }
 
-#[target.'cfg(not(target_env = "msvc"))'.dependencies]
-#tikv-jemallocator.workspace = true
+[target.'cfg(target_os = "macos")'.dependencies]
+tikv-jemallocator.workspace = true

python/src/lib.rs

@@ -10,6 +10,12 @@ use raphtory::python::{
 };
 use raphtory_graphql::python::pymodule::base_graphql_module;
 
+#[cfg(target_os = "macos")]
+use tikv_jemallocator::Jemalloc;
+#[cfg(target_os = "macos")]
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
 /// Raphtory graph analytics library
 #[pymodule]
 fn _raphtory(py: Python<'_>, m: &Bound<PyModule>) -> PyResult<()> {

raphtory/Cargo.toml

@@ -23,6 +23,7 @@ storage.workspace = true
 iter-enum = { workspace = true, features = ["rayon"] }
 hashbrown = { workspace = true }
 chrono = { workspace = true }
+crossbeam-channel = { workspace = true }
 itertools = { workspace = true }
 num-traits = { workspace = true }
 num-integer = { workspace = true }

raphtory/examples/eth_loader.rs

@@ -1,8 +1,6 @@
 #[cfg(feature = "io")]
-use raphtory::io::{
-    arrow::df_loaders::edges::ColumnNames, parquet_loaders::load_edges_from_parquet,
-};
-use raphtory::{errors::GraphError, prelude::*};
+use raphtory::io::parquet_loaders::load_edges_from_parquet;
+use raphtory::{arrow_loader::df_loaders::edges::ColumnNames, errors::GraphError, prelude::*};
 use std::path::{Path, PathBuf};
 
 /// Load ETH data from Parquet files into a Raphtory Graph.

raphtory/examples/snb_loader.rs

@@ -1,9 +1,6 @@
 #[cfg(feature = "io")]
-use raphtory::io::{
-    arrow::df_loaders::edges::ColumnNames,
-    parquet_loaders::{load_edges_from_parquet, load_nodes_from_parquet},
-};
-use raphtory::{errors::GraphError, prelude::*};
+use raphtory::io::parquet_loaders::{load_edges_from_parquet, load_nodes_from_parquet};
+use raphtory::{arrow_loader::df_loaders::edges::ColumnNames, errors::GraphError, prelude::*};
 use std::path::{Path, PathBuf};
 
 /// Construct the path to a named Parquet file inside `parquet_dir`.

raphtory/src/io/arrow/dataframe.rs → raphtory/src/arrow_loader/dataframe.rs

@@ -1,7 +1,7 @@
 use crate::{
     api::core::utils::time::TryIntoTime,
+    arrow_loader::node_col::{lift_node_col, NodeCol},
     errors::{into_load_err, GraphError, LoadError},
-    io::arrow::node_col::{lift_node_col, NodeCol},
 };
 use arrow::{
     array::{cast::AsArray, Array, ArrayRef, PrimitiveArray},

raphtory/src/io/arrow/df_loaders/edge_props.rs → raphtory/src/arrow_loader/df_loaders/edge_props.rs

@@ -1,10 +1,10 @@
 #[cfg(feature = "progress")]
-use crate::io::arrow::df_loaders::build_progress_bar;
+use crate::arrow_loader::df_loaders::build_progress_bar;
+#[cfg(feature = "progress")]
+use kdam::BarExt;
 
 use crate::{
-    db::api::view::StaticGraphViewOps,
-    errors::{into_graph_err, GraphError, LoadError},
-    io::arrow::{
+    arrow_loader::{
         dataframe::{DFChunk, DFView},
         df_loaders::{
             edges::{get_or_resolve_node_vids, store_node_ids, ColumnNames},
@@ -13,13 +13,14 @@ use crate::{
         layer_col::lift_layer_col,
         prop_handler::*,
     },
+    db::api::view::StaticGraphViewOps,
+    errors::{into_graph_err, GraphError, LoadError},
     prelude::*,
 };
 use arrow::{array::AsArray, datatypes::UInt64Type};
 use bytemuck::checked::cast_slice_mut;
 use db4_graph::WriteLockedGraph;
 use itertools::izip;
-use kdam::BarExt;
 use raphtory_api::{
     atomic_extra::atomic_usize_from_mut_slice,
     core::entities::{properties::prop::AsPropRef, LayerId, EID},