From 3de360946dbc44cda11c288f05f70e6381e40f59 Mon Sep 17 00:00:00 2001 From: skep Date: Sun, 8 Jun 2025 14:01:13 +0000 Subject: [PATCH 1/3] fix: explicit short param names --- helix-cli/src/args.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helix-cli/src/args.rs b/helix-cli/src/args.rs index ce5d01e50..d94b14c19 100644 --- a/helix-cli/src/args.rs +++ b/helix-cli/src/args.rs @@ -69,13 +69,13 @@ pub enum CommandType { #[derive(Debug, Args)] #[clap(name = "deploy", about = "Deploy a Helix project")] pub struct DeployCommand { - #[clap(short, long, help = "The path to the project")] + #[clap(short = 'P', long, help = "The path to the project")] pub path: Option, #[clap(short, long, help = "The output path")] pub output: Option, - #[clap(short, long, help = "Port to run the instance on")] + #[clap(short = 'p', long, help = "Port to run the instance on")] pub port: Option, } From 7ca934ffe9748f788594765d9f0257d0f4091f2f Mon Sep 17 00:00:00 2001 From: skep Date: Sun, 8 Jun 2025 20:45:42 +0000 Subject: [PATCH 2/3] WIP: storage abstraction --- helixdb/Cargo.toml | 8 +- helixdb/src/helix_engine/bm25/bm25.rs | 123 ++-- .../src/helix_engine/graph_core/graph_core.rs | 656 +---------------- .../graph_core/ops/bm25/search_bm25.rs | 64 +- helixdb/src/helix_engine/graph_core/ops/g.rs | 52 +- .../helix_engine/graph_core/ops/in_/in_.rs | 134 +--- .../helix_engine/graph_core/ops/in_/in_e.rs | 96 +-- .../helix_engine/graph_core/ops/in_/to_n.rs | 46 +- .../helix_engine/graph_core/ops/out/from_n.rs | 47 +- .../helix_engine/graph_core/ops/out/out.rs | 130 +--- .../helix_engine/graph_core/ops/out/out_e.rs | 119 +-- .../graph_core/ops/source/add_e.rs | 149 +--- .../graph_core/ops/source/add_n.rs | 106 +-- .../graph_core/ops/source/e_from_id.rs | 29 +- .../graph_core/ops/source/e_from_type.rs | 22 +- .../graph_core/ops/source/n_from_id.rs | 28 +- .../graph_core/ops/source/n_from_index.rs | 106 +-- .../graph_core/ops/source/n_from_type.rs | 24 +- .../src/helix_engine/graph_core/ops/tr_val.rs | 2 + .../helix_engine/graph_core/ops/util/dedup.rs | 13 +- .../helix_engine/graph_core/ops/util/drop.rs | 45 +- .../graph_core/ops/util/filter_mut.rs | 27 +- .../graph_core/ops/util/filter_ref.rs | 49 +- .../helix_engine/graph_core/ops/util/map.rs | 50 +- .../helix_engine/graph_core/ops/util/paths.rs | 167 ++--- .../helix_engine/graph_core/ops/util/props.rs | 19 +- .../helix_engine/graph_core/ops/util/range.rs | 15 +- .../graph_core/ops/util/update.rs | 117 +-- .../ops/vectors/brute_force_search.rs | 35 +- .../graph_core/ops/vectors/insert.rs | 114 ++- .../graph_core/ops/vectors/search.rs | 70 +- .../helix_engine/graph_core/traversal_iter.rs | 29 +- helixdb/src/helix_engine/mod.rs | 1 - .../helix_engine/storage_core/storage_core.rs | 440 ----------- .../storage_core/storage_methods.rs | 60 -- helixdb/src/helix_engine/types.rs | 42 +- helixdb/src/helix_engine/vector_core/hnsw.rs | 6 +- .../helix_engine/vector_core/vector_core.rs | 42 +- .../helix_gateway/connection/connection.rs | 3 +- helixdb/src/helix_gateway/gateway.rs | 3 +- helixdb/src/helix_gateway/mcp/mcp.rs | 42 +- helixdb/src/helix_gateway/mcp/tools.rs | 47 +- helixdb/src/helix_gateway/router/router.rs | 21 +- .../helix_gateway/thread_pool/thread_pool.rs | 5 +- helixdb/src/helix_storage/lmdb_storage.rs | 682 ++++++++++++++++++ helixdb/src/helix_storage/mod.rs | 131 ++++ helixdb/src/lib.rs | 1 + 47 files changed, 1650 insertions(+), 2567 deletions(-) delete mode 100644 helixdb/src/helix_engine/storage_core/storage_core.rs delete mode 100644 helixdb/src/helix_engine/storage_core/storage_methods.rs create mode 100644 helixdb/src/helix_storage/lmdb_storage.rs create mode 100644 helixdb/src/helix_storage/mod.rs diff --git a/helixdb/Cargo.toml b/helixdb/Cargo.toml index 8cb23c8cf..5639e22d6 100644 --- a/helixdb/Cargo.toml +++ b/helixdb/Cargo.toml @@ -12,7 +12,7 @@ tokio = { version = "1.44.2", features = ["full"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.110" bincode = "1.3.3" # TODO: Figure out bincode 2 impl with current serde impl -sonic-rs = "0.5.0" +sonic-rs = { version = "0.3.1", features = ["base64-simd"] } inventory = "0.3.16" twox-hash = "2.1.0" heed3 = "0.22.0" @@ -47,6 +47,9 @@ tokio-postgres = { version = "0.7", features = [ tempfile = { version = "3.2", optional = true } +bytes = "1.4.0" +thiserror = "1.0" + [dev-dependencies] rand = "0.9.0" lazy_static = "1.4.0" @@ -74,3 +77,6 @@ default = ["full"] strip = "debuginfo" lto = true opt-level = "z" + +[build-dependencies] +lalrpop = "0.20.0" diff --git a/helixdb/src/helix_engine/bm25/bm25.rs b/helixdb/src/helix_engine/bm25/bm25.rs index 96038823b..d21264b40 100644 --- a/helixdb/src/helix_engine/bm25/bm25.rs +++ b/helixdb/src/helix_engine/bm25/bm25.rs @@ -4,7 +4,7 @@ use std::{borrow::Cow, collections::HashMap, sync::Arc}; use crate::{ helix_engine::{ - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, + // storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, vector_core::{hnsw::HNSW, vector::HVector}, }, @@ -319,31 +319,10 @@ impl BM25 for HBM25Config { let k1 = 1.2; let b = 0.75; - // Ensure we don't have division by zero - let df = df.max(1); - let total_docs = total_docs.max(1); - - // Calculate IDF: log((N - df + 0.5) / (df + 0.5)) - // This can be negative when df is high relative to N, which is mathematically correct - let idf = ((total_docs as f64 - df as f64 + 0.5) / (df as f64 + 0.5)).ln(); - - // Ensure avgdl is not zero - let avgdl = if avgdl > 0.0 { - avgdl - } else { - doc_length as f64 - }; - - // Calculate BM25 score - let tf_component = (tf as f64 * (k1 as f64 + 1.0)) - / (tf as f64 + k1 as f64 * (1.0 - b as f64 + b as f64 * (doc_length as f64 / avgdl))); - - let score = (idf * tf_component) as f32; - - // The score can be negative when IDF is negative (term appears in most documents) - // This is mathematically correct - such terms have low discriminative power - // But documents with higher tf should still score higher than those with lower tf - score + let idf = ((total_docs as f64 - df as f64 + 0.5) / (df as f64 + 0.5)).ln_1p(); + let numerator = tf as f64 * (k1 as f64 + 1.0); + let denominator = tf as f64 + k1 as f64 * (1.0 - b as f64 + b as f64 * (doc_length as f64 / avgdl)); + (idf * numerator / denominator) as f32 } } @@ -359,52 +338,52 @@ pub trait HybridSearch { ) -> Result, GraphError>; } -impl HybridSearch for HelixGraphStorage { - fn hybrid_search( - &self, - txn: &RoTxn, - query: &str, - query_vector: &[f64], - vector_query: Option<&[f32]>, - alpha: f32, - limit: usize, - ) -> Result, GraphError> { - // Get BM25 scores - let bm25_results = self.bm25.search(txn, query, limit * 2)?; // Get more results for better fusion - let mut combined_scores: HashMap = HashMap::new(); - - // Add BM25 scores (weighted by alpha) - for (doc_id, score) in bm25_results { - combined_scores.insert(doc_id, alpha * score); - } - - // Add vector similarity scores if provided (weighted by 1-alpha) - if let Some(_query_vector) = vector_query { - // This would integrate with your existing vector search - // For now, we'll just use BM25 scores - // You would call your vector similarity search here and combine scores - let vector_results = self.vectors.search:: bool>( - txn, - query_vector, - limit * 2, - None, - false, - )?; - for doc in vector_results { - let doc_id = doc.id; - let score = doc.distance.unwrap_or(0.0); - combined_scores.insert(doc_id, (1.0 - alpha) * score as f32); - } - } - - // Sort by combined score and return top results - let mut results: Vec<(u128, f32)> = combined_scores.into_iter().collect(); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); - results.truncate(limit); - - Ok(results) - } -} +// impl HybridSearch for HelixGraphStorage { +// fn hybrid_search( +// &self, +// txn: &RoTxn, +// query: &str, +// query_vector: &[f64], +// vector_query: Option<&[f32]>, +// alpha: f32, +// limit: usize, +// ) -> Result, GraphError> { +// // Get BM25 scores +// let bm25_results = self.bm25.search(txn, query, limit * 2)?; // Get more results for better fusion +// let mut combined_scores: HashMap = HashMap::new(); + +// // Add BM25 scores (weighted by alpha) +// for (doc_id, score) in bm25_results { +// combined_scores.insert(doc_id, alpha * score); +// } + +// // Add vector similarity scores if provided (weighted by 1-alpha) +// if let Some(_query_vector) = vector_query { +// // This would integrate with your existing vector search +// // For now, we'll just use BM25 scores +// // You would call your vector similarity search here and combine scores +// let vector_results = self.vectors.search:: bool>( +// txn, +// query_vector, +// limit * 2, +// None, +// false, +// )?; +// for doc in vector_results { +// let doc_id = doc.id; +// let score = doc.distance.unwrap_or(0.0); +// combined_scores.insert(doc_id, (1.0 - alpha) * score as f32); +// } +// } + +// // Sort by combined score and return top results +// let mut results: Vec<(u128, f32)> = combined_scores.into_iter().collect(); +// results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)); +// results.truncate(limit); + +// Ok(results) +// } +// } pub trait BM25Flatten { fn flatten_bm25(&self) -> String; diff --git a/helixdb/src/helix_engine/graph_core/graph_core.rs b/helixdb/src/helix_engine/graph_core/graph_core.rs index 061c8618e..2dfb1167c 100644 --- a/helixdb/src/helix_engine/graph_core/graph_core.rs +++ b/helixdb/src/helix_engine/graph_core/graph_core.rs @@ -1,7 +1,6 @@ -use crate::helix_engine::storage_core::storage_core::HelixGraphStorage; -use crate::helix_engine::storage_core::storage_methods::StorageMethods; use crate::helix_engine::types::GraphError; use crate::helix_gateway::mcp::mcp::{McpBackend, McpConnections}; +use crate::helix_storage::{lmdb_storage::LmdbStorage, Storage}; use crate::props; use crate::protocol::filterable::{Filterable, FilterableType}; use crate::protocol::remapping::{Remapping, ResponseRemapping}; @@ -32,9 +31,8 @@ pub enum QueryInput { BooleanValue { value: bool }, } -pub struct HelixGraphEngine { - // TODO: is there a reason for this? - pub storage: Arc, +pub struct HelixGraphEngine { + pub storage: Arc, pub mcp_backend: Option>, pub mcp_connections: Option>>, } @@ -59,13 +57,11 @@ impl HelixGraphEngineOpts { } } -impl HelixGraphEngine { - pub fn new(opts: HelixGraphEngineOpts) -> Result { +impl HelixGraphEngine { + pub fn new(opts: HelixGraphEngineOpts) -> Result, GraphError> { let should_use_mcp = opts.config.mcp; - let storage = match HelixGraphStorage::new(opts.path.as_str(), opts.config) { - Ok(db) => Arc::new(db), - Err(err) => return Err(err), - }; + let storage = Arc::new(LmdbStorage::new(opts.path.as_str(), opts.config)?); + let (mcp_backend, mcp_connections) = if should_use_mcp { let mcp_backend = Arc::new(McpBackend::new(storage.clone())); let mcp_connections = Arc::new(Mutex::new(McpConnections::new())); @@ -79,644 +75,10 @@ impl HelixGraphEngine { mcp_connections, }) } +} - // pub fn print_result_as_json(&self, traversal: &TraversalBuilder) { - // let current_step = &traversal.current_step; - // let json_result = json!(current_step); - // println!("{}", json_result.to_string()); - // } - - // pub fn print_result_as_pretty_json(&self, traversal: &TraversalBuilder) { - // let current_step = &traversal.current_step; - // let json_result = json!(current_step); - // println!("{}", serde_json::to_string_pretty(&json_result).unwrap()); - // } - - // /// implement error for this function - // pub fn result_to_json(&self, traversal: &TraversalBuilder) -> Vec { - // let current_step = &traversal.current_step; - // let mut json_string = serde_json::to_string(current_step).unwrap(); - // json_string.push_str("\n"); - // json_string.into_bytes() - // } - - // pub fn result_to_json_string(&self, traversal: &TraversalBuilder) -> String { - // let current_step = &traversal.current_step; - // let mut json_string = serde_json::to_string(current_step).unwrap(); - // json_string.push_str("\n"); - // json_string - // } - +impl HelixGraphEngine { pub fn query(&self, query: String, params: Vec) -> Result { Ok(String::new()) } - // let ast: Source = match HelixParser::parse_source(query.as_str()) { - // Ok(src) => src, - // Err(err) => return Err(GraphError::from(err)), - // }; - // let mut return_vals: HashMap = HashMap::new(); - // let vars: Arc>> = Arc::new(RwLock::new(HashMap::new())); - // // let mut results = Vec::with_capacity(return_vals.len()); - - // for query in ast.queries { - // for stmt in query.statements { - // match stmt { - // Statement::Assignment(ass) => { - // let value: ReturnValue = match ass.value { - // Expression::Traversal(tr) | Expression::Exists(tr) => { - // // build traversal based on steps with traversal builder - // // initialise from start node - // // step through all steps and execute. - // self.evaluate_traversal( - // tr, - // Arc::clone(&vars), - // TraversalValue::Empty, - // )? - // } - // Expression::AddVertex(add_v) => { - // let mut txn = self.storage.graph_env.write_txn()?; - // let mut tr_builder = TraversalBuilder::new( - // Arc::clone(&self.storage), - // TraversalValue::Empty, - // ); - // let label = match add_v.vertex_type { - // Some(l) => l, - // None => String::new(), - // }; - // let props = match add_v.fields { - // Some(p) => p, - // None => props! {}, - // }; - // tr_builder.add_v(&mut txn, label.as_str(), props, None); - // let result = tr_builder.result(txn)?; - // ReturnValue::TraversalValues(result) - // } - // Expression::AddEdge(add_e) => { - // let mut txn = self.storage.graph_env.write_txn()?; - // let mut tr_builder = TraversalBuilder::new( - // Arc::clone(&self.storage), - // TraversalValue::Empty, - // ); - // let label = match add_e.edge_type { - // Some(l) => l, - // None => String::new(), - // }; - // let props = match add_e.fields { - // Some(p) => p, - // None => props! {}, - // }; - // tr_builder.add_e( - // &mut txn, - // label.as_str(), - // &Self::id_type_to_id( - // add_e.connection.from_id, - // Arc::clone(&vars), - // )?, - // &Self::id_type_to_id( - // add_e.connection.to_id, - // Arc::clone(&vars), - // )?, - // props, - // ); - // let result = tr_builder.result(txn)?; - // ReturnValue::TraversalValues(result) - // } - // _ => { - // // insert variable to hashmap - // let var = - // match ass.value { - // Expression::StringLiteral(value) => TraversalValue::from(( - // ass.variable.clone(), - // Value::String(value), - // )), - // Expression::IntegerLiteral(value) => TraversalValue::from( - // (ass.variable.clone(), Value::Integer(value)), - // ), - // Expression::FloatLiteral(value) => TraversalValue::from(( - // ass.variable.clone(), - // Value::Float(value), - // )), - // Expression::BooleanLiteral(value) => TraversalValue::from( - // (ass.variable.clone(), Value::Boolean(value)), - // ), - // _ => unreachable!(), - // }; - // ReturnValue::TraversalValues(var) - // } - // }; - - // vars.write().unwrap().insert(ass.variable, value); - // } - // Statement::AddVertex(add_v) => { - // let mut txn = self.storage.graph_env.write_txn()?; - // let mut tr_builder = - // TraversalBuilder::new(Arc::clone(&self.storage), TraversalValue::Empty); - // let label = add_v.vertex_type.unwrap_or_default(); - // let props = add_v.fields.unwrap_or_default(); - // tr_builder.add_v(&mut txn, label.as_str(), props, None); - // tr_builder.execute()?; - // } - // Statement::AddEdge(add_e) => { - // let mut txn = self.storage.graph_env.write_txn()?; - // let mut tr_builder = - // TraversalBuilder::new(Arc::clone(&self.storage), TraversalValue::Empty); - - // let label = add_e.edge_type.unwrap_or_default(); - // let props = add_e.fields.unwrap_or_default(); - - // tr_builder.add_e( - // &mut txn, - // label.as_str(), - // &Self::id_type_to_id(add_e.connection.from_id, Arc::clone(&vars))?, - // &Self::id_type_to_id(add_e.connection.to_id, Arc::clone(&vars))?, - // props, - // ); - // tr_builder.execute()?; - // } - // Statement::Drop(drop) => {} - // } - // } - // for return_value in query.return_values { - // match return_value { - // Expression::Identifier(var_name) => { - // if let Some(val) = vars.read().unwrap().get(&var_name) { - // return_vals.insert(var_name, val.clone()); // fix clone - // } - // } - // Expression::Traversal(tr) => { - // let var_name = match tr.start { - // StartNode::Variable(var_name) => var_name, - // _ => { - // return Err(GraphError::from("Return value must be a variable!")); - // } - // }; - // if let Some(val) = vars.read().unwrap().get(&var_name) { - // return_vals.insert(var_name, val.clone()); // fix clone - // } - // } - // _ => { - // return Err(GraphError::from("Return value must be a variable!")); - // } - // } - // } - // } - - // let json_string = sonic_rs::to_string_pretty(&return_vals).unwrap(); - // Ok(json_string) - // } - - // fn evaluate_traversal( - // &self, - // tr: Box, - // vars: Arc>>, - // anon_start: TraversalValue, - // ) -> Result { - // let start_nodes: TraversalValue = match tr.start { - // StartNode::Vertex { types, ids } | StartNode::Edge { types, ids } => { - // let types = match types { - // Some(types) => types, - // None => vec![], - // }; - // let ids = match ids { - // Some(ids) => ids, - // None => vec![], - // }; - // let mut txn = self.storage.graph_env.read_txn()?; - // let mut start_tr = - // TraversalBuilder::new(Arc::clone(&self.storage), TraversalValue::Empty); - // match ids.len() { - // 0 => match types.len() { - // 0 => start_tr.v(&txn), - // _ => start_tr.v_from_types(&txn, &types), - // }, - // _ => start_tr.v_from_ids(&txn, &ids), - // }; - // start_tr.result(txn)? - // } - // StartNode::Variable(var_name) => match vars.read().unwrap().get(&var_name) { - // Some(vals) => match vals.clone() { - // ReturnValue::TraversalValues(vals) => vals, - // _ => unreachable!(), - // }, - // None => { - // return Err(GraphError::from(format!( - // "Variable: {} not found!", - // var_name - // ))) - // } - // }, - // StartNode::Anonymous => anon_start, - // _ => unreachable!(), - // }; - - // let mut txn = self.storage.graph_env.read_txn()?; - // let mut tr_builder = TraversalBuilder::new(Arc::clone(&self.storage), start_nodes); - - // for step in &tr.steps { - // match step { - // Step::Vertex(graph_step) => match graph_step { - // GraphStep::Out(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.out(&txn, label); - // } - // } - // None => { - // tr_builder.out(&txn, ""); - // } - // }, - // GraphStep::In(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.in_(&txn, label); - // } - // } - // None => { - // tr_builder.in_(&txn, ""); - // } - // }, - // GraphStep::OutE(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.out_e(&txn, label); - // } - // } - // None => { - // tr_builder.out_e(&txn, ""); - // } - // }, - // GraphStep::InE(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.in_e(&txn, label); - // } - // } - // None => { - // tr_builder.in_e(&txn, ""); - // } - // }, - // GraphStep::Both(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.both(&txn, label); - // } - // } - // None => { - // tr_builder.both(&txn, ""); - // } - // }, - // GraphStep::BothE(labels) => match labels { - // Some(l) => { - // if l.len() > 1 { - // return Err(GraphError::from("Cannot use more than 1 label yet! This feature will be coming soon.")); - // } - // if let Some(label) = l.first() { - // tr_builder.both_e(&txn, label); - // } - // } - // None => { - // tr_builder.both_e(&txn, ""); - // } - // }, - // _ => unreachable!(), - // }, - // Step::Edge(graph_step) => match graph_step { - // GraphStep::OutV => { - // tr_builder.out_v(&txn); - // } - // GraphStep::InV => { - // tr_builder.in_v(&txn); - // } - // GraphStep::BothV => { - // tr_builder.both_v(&txn); - // } - // _ => unreachable!(), - // }, - // Step::Count => { - // tr_builder.count(); - // } - // Step::Props(property_names) => { - // assert!(property_names.len() > 0, "Property names must be provided!"); - // tr_builder.get_properties(&txn, property_names); - // } - // Step::Where(expression) => { - // match &**expression { - // Expression::Traversal(anon_tr) => match anon_tr.start { - // StartNode::Anonymous => match tr_builder.current_step { - // TraversalValue::NodeArray(_) => { - // tr_builder.filter_nodes(&txn, |val| { - // match self.evaluate_traversal( - // anon_tr.clone(), - // Arc::clone(&vars), - // TraversalValue::from(val), - // )? { - // ReturnValue::Boolean(val) => Ok(val), - // _ => { - // return Err(GraphError::from( - // "Where clause must evaluate to a boolean!", - // )); - // } - // } - // }); - // } - // TraversalValue::EdgeArray(_) => { - // tr_builder.filter_edges(&txn, |val| { - // match self.evaluate_traversal( - // anon_tr.clone(), - // Arc::clone(&vars), - // TraversalValue::from(val), - // )? { - // ReturnValue::Boolean(res) => Ok(res), - // _ => { - // return Err(GraphError::from( - // "Where clause must evaluate to a boolean!", - // )); - // } - // } - // }); - // } - // _ => { - // return Err(GraphError::from( - // format!("Exists clause must follow a traversal step! Got step: {:?}", anon_tr.start ), - // )); - // } - // }, - // _ => { - // return Err(GraphError::from("Where clause must start with an anonymous traversal or exists query!")); - // } - // }, - - // Expression::Exists(anon_tr) => match anon_tr.start { - // StartNode::Anonymous => match tr_builder.current_step { - // TraversalValue::NodeArray(_) => { - // tr_builder.filter_nodes(&txn, |val| { - // match self.evaluate_traversal( - // anon_tr.clone(), - // Arc::clone(&vars), - // TraversalValue::from(val), - // )? { - // ReturnValue::Boolean(val) => Ok(val), - // _ => { - // return Err(GraphError::from( - // "Where clause must evaluate to a boolean!", - // )); - // } - // } - // }); - // } - // TraversalValue::EdgeArray(_) => { - // tr_builder.filter_edges(&txn, |val| { - // match self.evaluate_traversal( - // anon_tr.clone(), - // Arc::clone(&vars), - // TraversalValue::from(val), - // )? { - // ReturnValue::Boolean(res) => Ok(res), - // _ => { - // return Err(GraphError::from( - // "Where clause must evaluate to a boolean!", - // )); - // } - // } - // }); - // } - // _ => { - // return Err(GraphError::from( - // "Exists clause must follow a traversal step!", - // )); - // } - // }, - // _ => { - // return Err(GraphError::from("Where clause must start with an anonymous traversal or exists query!")); - // } - // }, - // _ => { - // return Err(GraphError::from("Where clause must start with an anonymous traversal or exists query!")); - // } - // } - // } - - // Step::BooleanOperation(op) => { - // // let previous_step = tr.steps[index - 1].clone(); - // // match previous_step { - // // Step::Count => {} - // // Step::Props(_) => {} - // // _ => { - // // return Err(GraphError::from(format!( - // // "Boolean operation must follow a traversal step! Got step: {:?}", - // // previous_step - // // ))); - // // } - // // }; - - // match tr_builder.current_step { - // TraversalValue::Count(count) => { - // return Ok(ReturnValue::Boolean(Self::manage_int_bool_exp( - // op, - // count.value() as i32, - // ))) - // } - // TraversalValue::ValueArray(ref vals) => { - // let mut res = Vec::with_capacity(vals.len()); - // for (_, val) in vals { - // match val { - // Value::Integer(val) => { - // res.push(Self::manage_int_bool_exp(op, val.clone())); - // } - // Value::Float(val) => { - // res.push(Self::manage_float_bool_exp(op, val.clone())); - // } - // _ => { - // return Err(GraphError::from( - // "Expression should resolve to a number!", - // )); - // } - // } - // } - // return Ok(ReturnValue::Boolean(res.iter().all(|&x| x))); - // } - // _ => { - // return Err(GraphError::from( - // format!("Boolean operation must follow a count or numerical property step! Got step: {:?} for traversal {:?}", tr_builder.current_step, step), - // )); - // } - // }; - // } - // _ => unreachable!(), - // } - // } - // let result = tr_builder.result(txn)?; - // Ok(ReturnValue::TraversalValues(result)) - // } - - // fn manage_float_bool_exp(op: &BooleanOp, fl: f64) -> bool { - // match op { - // BooleanOp::GreaterThan(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl > val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::GreaterThanOrEqual(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl >= val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::LessThan(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl < val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::LessThanOrEqual(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl <= val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::Equal(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl == val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::NotEqual(expr) => match **expr { - // Expression::FloatLiteral(val) => { - // return fl != val; - // } - // _ => { - // return false; - // } - // }, - // _ => { - // return false; - // } - // }; - // } - - // fn manage_int_bool_exp(op: &BooleanOp, i: i32) -> bool { - // match op { - // BooleanOp::GreaterThan(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i > val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::GreaterThanOrEqual(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i >= val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::LessThan(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i < val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::LessThanOrEqual(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i <= val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::Equal(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i == val; - // } - // _ => { - // return false; - // } - // }, - // BooleanOp::NotEqual(expr) => match **expr { - // Expression::IntegerLiteral(val) => { - // return i != val; - // } - // _ => { - // return false; - // } - // }, - // _ => { - // return false; - // } - // }; - // } - - // fn id_type_to_id( - // id_type: IdType, - // vars: Arc>>, - // ) -> Result { - // match id_type { - // IdType::Literal(s) => Ok(s), - // IdType::Identifier(s) => { - // let reader = vars.read().unwrap(); - // let vals = reader.get(&s).unwrap(); - // match vals { - // ReturnValue::TraversalValues(tr) => { - // match tr { - // TraversalValue::NodeArray(arr) => { - // if arr.len() != 1 { - // // throw err - // return Err(GraphError::from(format!( - // "Node array too long, expected length 1 but got length {}", - // arr.len() - // ))); - // }; - // // get first and get id - // let node = arr.first().unwrap(); - // Ok(node.id.clone()) - // } - // TraversalValue::EdgeArray(arr) => { - // if arr.len() != 1 { - // // throw error - // return Err(GraphError::from(format!( - // "Edge array too long, expected length 1 but got length {}", - // arr.len() - // ))); - // }; - // let edge = arr.first().unwrap(); - // Ok(edge.id.clone().deref().to_string()) // change - // } - // _ => unreachable!(), - // } - // } - // _ => unreachable!(), - // } - // } - // } - // } } diff --git a/helixdb/src/helix_engine/graph_core/ops/bm25/search_bm25.rs b/helixdb/src/helix_engine/graph_core/ops/bm25/search_bm25.rs index 7e33c301f..f62c5b411 100644 --- a/helixdb/src/helix_engine/graph_core/ops/bm25/search_bm25.rs +++ b/helixdb/src/helix_engine/graph_core/ops/bm25/search_bm25.rs @@ -1,64 +1,63 @@ -use heed3::RoTxn; - use super::super::tr_val::TraversalVal; use crate::helix_engine::{ - bm25::bm25::BM25, - graph_core::traversal_iter::RoTraversalIterator, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, - types::{GraphError, VectorError}, - vector_core::{hnsw::HNSW, vector::HVector}, + bm25::bm25::BM25, graph_core::traversal_iter::RoTraversalIterator, types::GraphError, }; -use std::{iter::once, sync::Arc}; +use crate::helix_storage::lmdb_storage::LmdbStorage; +use crate::helix_storage::Storage; +use std::sync::Arc; -pub struct SearchBM25<'scope, 'inner> { - txn: &'scope RoTxn<'scope>, +pub struct SearchBM25<'scope, 'inner, S: Storage + ?Sized> { + txn: &'scope S::RoTxn<'scope>, iter: std::vec::IntoIter<(u128, f32)>, - storage: Arc, + storage: Arc, label: &'inner str, } -// implementing iterator for OutIterator -impl<'scope, 'inner> Iterator for SearchBM25<'scope, 'inner> { +impl<'scope, 'inner, S: Storage + ?Sized> Iterator for SearchBM25<'scope, 'inner, S> { type Item = Result; fn next(&mut self) -> Option { - let next = self.iter.next()?; - match self.storage.get_node(self.txn, &next.0) { - Ok(node) => { - if node.label == self.label { - Some(Ok(TraversalVal::Node(node))) - } else { - return None; - } - } + let next_item = self.iter.next()?; + match self.storage.get_node(self.txn, &next_item.0) { + Ok(node) if node.label == self.label => Some(Ok(TraversalVal::Node(node))), + Ok(_) => self.next(), // Continue searching if label doesn't match Err(e) => Some(Err(e)), } } } -pub trait SearchBM25Adapter<'a>: Iterator> { +pub trait SearchBM25Adapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn search_bm25( self, label: &str, query: &str, k: usize, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> SearchBM25Adapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> SearchBM25Adapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage = crate::helix_storage::lmdb_storage::LmdbRoTxn<'a>> + 'static, { fn search_bm25( self, label: &str, query: &str, k: usize, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let results = self - .storage - .bm25 - .search(self.txn, query, k) - .unwrap_or_default(); + ) -> RoTraversalIterator<'a, impl Iterator>, S> { + let results = if let Some(lmdb_storage) = + (self.storage.as_ref() as &dyn std::any::Any).downcast_ref::() + { + lmdb_storage + .bm25 + .search(&self.txn, query, k) + .unwrap_or_default() + } else { + Vec::new() + }; let iter = SearchBM25 { txn: self.txn, @@ -66,7 +65,6 @@ impl<'a, I: Iterator>> SearchBM25Adapter storage: Arc::clone(&self.storage), label, }; - // Wrap it with the RoTraversalIterator adapter RoTraversalIterator { inner: iter, storage: self.storage, diff --git a/helixdb/src/helix_engine/graph_core/ops/g.rs b/helixdb/src/helix_engine/graph_core/ops/g.rs index dd7223055..0f1428c63 100644 --- a/helixdb/src/helix_engine/graph_core/ops/g.rs +++ b/helixdb/src/helix_engine/graph_core/ops/g.rs @@ -1,10 +1,9 @@ use super::tr_val::TraversalVal; use crate::helix_engine::{ graph_core::traversal_iter::{RoTraversalIterator, RwTraversalIterator}, - storage_core::storage_core::HelixGraphStorage, types::GraphError, }; -use heed3::{RoTxn, RwTxn}; +use crate::helix_storage::Storage; use std::sync::Arc; pub struct G {} @@ -25,13 +24,10 @@ impl G { /// let traversal = G::new(storage, &txn); /// ``` #[inline] - pub fn new<'a>( - storage: Arc, - txn: &'a RoTxn<'a>, - ) -> RoTraversalIterator<'a, impl Iterator>> - where - Self: Sized, - { + pub fn new<'a, S: Storage + ?Sized>( + storage: Arc, + txn: &'a S::RoTxn<'a>, + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { inner: std::iter::once(Ok(TraversalVal::Empty)), storage, @@ -54,11 +50,11 @@ impl G { /// let txn = storage.graph_env.read_txn().unwrap(); /// let traversal = G::new_from(storage, &txn, vec![TraversalVal::Node(Node { id: 1, label: "Person".to_string(), properties: None })]); /// ``` - pub fn new_from<'a>( - storage: Arc, - txn: &'a RoTxn<'a>, + pub fn new_from<'a, S: Storage + ?Sized>( + storage: Arc, + txn: &'a S::RoTxn<'a>, items: Vec, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { inner: items.into_iter().map(|val| Ok(val)), storage, @@ -81,13 +77,15 @@ impl G { /// let txn = storage.graph_env.write_txn().unwrap(); /// let traversal = G::new_mut(storage, &mut txn); /// ``` - pub fn new_mut<'scope, 'env>( - storage: Arc, - txn: &'scope mut RwTxn<'env>, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>> - where - Self: Sized, - { + pub fn new_mut<'scope, 'env, S: Storage + ?Sized>( + storage: Arc, + txn: &'scope mut S::RwTxn<'env>, + ) -> RwTraversalIterator< + 'scope, + 'env, + impl Iterator>, + S, + > { RwTraversalIterator { inner: std::iter::once(Ok(TraversalVal::Empty)), storage, @@ -110,12 +108,16 @@ impl G { /// let txn = storage.graph_env.write_txn().unwrap(); /// let traversal = G::new_mut_from(storage, &mut txn, vec![TraversalVal::Node(Node { id: 1, label: "Person".to_string(), properties: None })]); /// ``` - pub fn new_mut_from<'scope, 'env>( - storage: Arc, - txn: &'scope mut RwTxn<'env>, + pub fn new_mut_from<'scope, 'env, S: Storage + ?Sized>( + storage: Arc, + txn: &'scope mut S::RwTxn<'env>, vals: Vec, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>> - { + ) -> RwTraversalIterator< + 'scope, + 'env, + impl Iterator>, + S, + > { RwTraversalIterator { inner: vals.into_iter().map(|val| Ok(val)), storage, diff --git a/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs b/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs index 238ef1dd4..9cc32371b 100644 --- a/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs +++ b/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs @@ -1,131 +1,69 @@ use crate::{ helix_engine::{ graph_core::{ - ops::{ - source::add_e::EdgeType, - tr_val::{Traversable, TraversalVal}, - }, + ops::tr_val::{Traversable, TraversalVal}, traversal_iter::RoTraversalIterator, }, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, - protocol::label_hash::hash_label, + helix_storage::Storage, }; use heed3::{types::Bytes, RoTxn}; use std::sync::Arc; -pub struct InNodesIterator<'a, T> { - pub iter: heed3::RoIter< - 'a, - Bytes, - heed3::types::LazyDecode, - heed3::iteration_method::MoveOnCurrentKeyDuplicates, - >, - pub storage: Arc, - pub txn: &'a T, - pub edge_type: &'a EdgeType, +pub struct InNodesIterator<'a, I, S: Storage + ?Sized> { + iter: I, + storage: Arc, + txn: &'a S::RoTxn<'a>, + edge_label: &'a str, } -impl<'a> Iterator for InNodesIterator<'a, RoTxn<'a>> { +impl<'a, I, S> Iterator for InNodesIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, +{ type Item = Result; fn next(&mut self) -> Option { - while let Some(Ok((_, data))) = self.iter.next() { - match data.decode() { - Ok(data) => { - let (node_id, _) = match HelixGraphStorage::unpack_adj_edge_data(&data) { - Ok(data) => data, - Err(e) => { - println!("Error unpacking edge data: {:?}", e); - return Some(Err(e)); - } - }; - match self.edge_type { - EdgeType::Node => { - if let Ok(node) = self.storage.get_node(self.txn, &node_id) { - return Some(Ok(TraversalVal::Node(node))); - } - } - EdgeType::Vec => { - if let Ok(vector) = self.storage.get_vector(self.txn, &node_id) { - return Some(Ok(TraversalVal::Vector(vector))); - } - } - } - } - Err(e) => { - println!("Error decoding edge data: {:?}", e); - return Some(Err(GraphError::DecodeError(e.to_string()))); - } - } - } - None + self.iter.next().map(|item| { + let start_node = item?; + let nodes = + self.storage + .get_in_nodes(self.txn, self.edge_label, &start_node.id())?; + Ok(TraversalVal::NodeArray(nodes)) + }) } } -pub trait InAdapter<'a, T>: Iterator> { - /// Returns an iterator containing the nodes that have an incoming edge with the given label. - /// - /// Note that the `edge_label` cannot be empty and must be a valid, existing edge label. - /// - /// To provide safety, you cannot get all incoming nodes as it would be ambiguous as to what - /// type that resulting node would be. +pub trait InAdapter<'a, S: Storage + ?Sized, I: Iterator>>: + Iterator> +{ fn in_( self, edge_label: &'a str, - edge_type: &'a EdgeType, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator> + 'a> InAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> InAdapter<'a, S, I> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { #[inline] fn in_( self, edge_label: &'a str, - edge_type: &'a EdgeType, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let db = Arc::clone(&self.storage); - let storage = Arc::clone(&self.storage); - let txn = self.txn; - let iter = self - .inner - .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::in_edge_key( - &match item { - Ok(item) => item.id(), - Err(_) => return None, - }, - &edge_label_hash, - ); - match db - .in_edges_db - .lazily_decode_data() - .get_duplicates(txn, &prefix) - { - Ok(Some(iter)) => Some(InNodesIterator { - iter, - storage: Arc::clone(&db), - txn, - edge_type, - }), - Ok(None) => None, - Err(e) => { - println!("Error getting in edges: {:?}", e); - // return Err(e); - None - } - } - }) - .flatten(); - + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { - inner: iter, - storage, - txn, + inner: InNodesIterator { + iter: self.inner, + storage: self.storage.clone(), + txn: self.txn, + edge_label, + }, + storage: self.storage, + txn: self.txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/in_/in_e.rs b/helixdb/src/helix_engine/graph_core/ops/in_/in_e.rs index 850a4e5d6..987eb1c35 100644 --- a/helixdb/src/helix_engine/graph_core/ops/in_/in_e.rs +++ b/helixdb/src/helix_engine/graph_core/ops/in_/in_e.rs @@ -1,57 +1,35 @@ +use std::sync::Arc; + use crate::{ helix_engine::{ graph_core::{ ops::tr_val::{Traversable, TraversalVal}, traversal_iter::RoTraversalIterator, }, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, - protocol::label_hash::hash_label, + helix_storage::Storage, + protocol::{items::Edge, label_hash::hash_label}, }; -use heed3::{types::Bytes, RoTxn}; -use std::sync::Arc; -pub struct InEdgesIterator<'a, T> { - pub iter: heed3::RoIter< - 'a, - Bytes, - heed3::types::LazyDecode, - heed3::iteration_method::MoveOnCurrentKeyDuplicates, - >, - pub storage: Arc, - pub txn: &'a T, +pub struct InEdgesIterator { + pub edges: Vec, + pub storage: Arc, } -impl<'a> Iterator for InEdgesIterator<'a, RoTxn<'a>> { +impl Iterator for InEdgesIterator { type Item = Result; fn next(&mut self) -> Option { - while let Some(Ok((_, data))) = self.iter.next() { - match data.decode() { - Ok(data) => { - let (_, edge_id) = match HelixGraphStorage::unpack_adj_edge_data(&data) { - Ok(data) => data, - Err(e) => { - println!("Error unpacking edge data: {:?}", e); - return Some(Err(e)); - } - }; - if let Ok(edge) = self.storage.get_edge(self.txn, &edge_id) { - return Some(Ok(TraversalVal::Edge(edge))); - } - } - Err(e) => { - println!("Error decoding edge data: {:?}", e); - return Some(Err(GraphError::DecodeError(e.to_string()))); - } - } - } - None + self.edges + .pop() + .map(|edge| Ok(TraversalVal::Edge(edge))) } } -pub trait InEdgesAdapter<'a, T>: Iterator> { +pub trait InEdgesAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ /// Returns an iterator containing the edges that have an incoming edge with the given label. /// /// Note that the `edge_label` cannot be empty and must be a valid, existing edge label. @@ -61,51 +39,31 @@ pub trait InEdgesAdapter<'a, T>: Iterator RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> InEdgesAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> InEdgesAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline] fn in_e( self, edge_label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let db = Arc::clone(&self.storage); + ) -> RoTraversalIterator<'a, impl Iterator>, S> { let storage = Arc::clone(&self.storage); let txn = self.txn; let iter = self .inner - .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); - - let prefix = HelixGraphStorage::in_edge_key( - &match item { - Ok(item) => item.id(), - Err(_) => return None, - }, - &edge_label_hash, - ); - match db - .in_edges_db - .lazily_decode_data() - .get_duplicates(txn, &prefix) - { - Ok(Some(iter)) => Some(InEdgesIterator { - iter, - storage: Arc::clone(&db), - txn, - }), - Ok(None) => None, - Err(e) => { - println!("Error getting in edges: {:?}", e); - // return Err(e); - None - } - } + .filter_map(move |item| match item { + Ok(item) => Some(item.id()), + Err(_) => None, }) - .flatten(); + .map(move |id| self.storage.get_in_edges(txn, edge_label, &id)) + .filter_map(|res| res.ok()) + .flatten() + .map(|edge| Ok(TraversalVal::Edge(edge))); RoTraversalIterator { inner: iter, diff --git a/helixdb/src/helix_engine/graph_core/ops/in_/to_n.rs b/helixdb/src/helix_engine/graph_core/ops/in_/to_n.rs index 3e5907000..157b5f08f 100644 --- a/helixdb/src/helix_engine/graph_core/ops/in_/to_n.rs +++ b/helixdb/src/helix_engine/graph_core/ops/in_/to_n.rs @@ -1,55 +1,49 @@ use crate::helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }; -use heed3::RoTxn; +use crate::helix_storage::Storage; use std::sync::Arc; -pub struct ToNIterator<'a, I, T> { +pub struct ToNIterator<'a, I, T, S: Storage + ?Sized> { iter: I, - storage: Arc, + storage: Arc, txn: &'a T, } -// implementing iterator for OutIterator -impl<'a, I> Iterator for ToNIterator<'a, I, RoTxn<'a>> +impl<'a, I, S: Storage + ?Sized> Iterator for ToNIterator<'a, I, S::RoTxn<'a>, S> where I: Iterator>, { type Item = Result; fn next(&mut self) -> Option { - match self.iter.next() { - Some(item) => match item { - Ok(TraversalVal::Edge(item)) => Some(Ok(TraversalVal::Node( - match self.storage.get_node(self.txn, &item.to_node) { - Ok(node) => node, - Err(e) => { - println!("Error getting node: {:?}", e); - return Some(Err(e)); - } - }, - ))), - _ => return None, - }, - None => None, - } + self.iter.next().and_then(|item| match item { + Ok(TraversalVal::Edge(edge)) => { + Some(self.storage.get_node(self.txn, &edge.to_node).map(TraversalVal::Node)) + } + _ => None, + }) } } -pub trait ToNAdapter<'a, T>: Iterator> { + +pub trait ToNAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn to_n( self, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> ToNAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> ToNAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline(always)] fn to_n( self, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { let iter = ToNIterator { iter: self.inner, storage: Arc::clone(&self.storage), diff --git a/helixdb/src/helix_engine/graph_core/ops/out/from_n.rs b/helixdb/src/helix_engine/graph_core/ops/out/from_n.rs index a5dd8e2dc..9b1a532be 100644 --- a/helixdb/src/helix_engine/graph_core/ops/out/from_n.rs +++ b/helixdb/src/helix_engine/graph_core/ops/out/from_n.rs @@ -1,56 +1,51 @@ use crate::helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }; -use heed3::RoTxn; +use crate::helix_storage::Storage; use std::sync::Arc; -pub struct FromNIterator<'a, I, T> { +pub struct FromNIterator<'a, I, T, S: Storage + ?Sized> { iter: I, - storage: Arc, + storage: Arc, txn: &'a T, } -impl<'a, I> Iterator for FromNIterator<'a, I, RoTxn<'a>> +impl<'a, I, S: Storage + ?Sized> Iterator for FromNIterator<'a, I, S::RoTxn<'a>, S> where I: Iterator>, { type Item = Result; fn next(&mut self) -> Option { - match self.iter.next() { - Some(item) => match item { - Ok(TraversalVal::Edge(item)) => Some(Ok(TraversalVal::Node( - match self.storage.get_node(self.txn, &item.from_node) { - Ok(node) => node, - Err(e) => { - println!("Error getting node: {:?}", e); - return Some(Err(e)); - } - }, - ))), - _ => return None, - }, - None => None, - } + self.iter.next().and_then(|item| match item { + Ok(TraversalVal::Edge(edge)) => Some( + self.storage + .get_node(self.txn, &edge.from_node) + .map(TraversalVal::Node), + ), + _ => None, + }) } } -pub trait FromNAdapter<'a, T>: Iterator> { - /// Returns an iterator containing the nodes that the edges in `self.inner` originate from. +pub trait FromNAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn from_n( self, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator> + 'a> FromNAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> FromNAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { #[inline(always)] fn from_n( self, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { let iter = FromNIterator { iter: self.inner, storage: Arc::clone(&self.storage), diff --git a/helixdb/src/helix_engine/graph_core/ops/out/out.rs b/helixdb/src/helix_engine/graph_core/ops/out/out.rs index b402945fe..f6889b687 100644 --- a/helixdb/src/helix_engine/graph_core/ops/out/out.rs +++ b/helixdb/src/helix_engine/graph_core/ops/out/out.rs @@ -7,126 +7,66 @@ use crate::{ }, traversal_iter::RoTraversalIterator, }, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, - protocol::label_hash::hash_label, + helix_storage::Storage, }; use heed3::{types::Bytes, RoTxn, WithTls}; use std::sync::Arc; -pub struct OutNodesIterator<'a, T> { - pub iter: heed3::RoIter< - 'a, - Bytes, - heed3::types::LazyDecode, - heed3::iteration_method::MoveOnCurrentKeyDuplicates, - >, - pub storage: Arc, - pub edge_type: &'a EdgeType, - pub txn: &'a T, +pub struct OutNodesIterator<'a, I, S: Storage + ?Sized> { + iter: I, + storage: Arc, + txn: &'a S::RoTxn<'a>, + edge_label: &'a str, } -impl<'a> Iterator for OutNodesIterator<'a, RoTxn<'a>> { +impl<'a, I, S> Iterator for OutNodesIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, +{ type Item = Result; fn next(&mut self) -> Option { - while let Some(Ok((_, data))) = self.iter.next() { - match data.decode() { - Ok(data) => { - let (item_id, _) = match HelixGraphStorage::unpack_adj_edge_data(&data) { - Ok(data) => data, - Err(e) => { - println!("Error unpacking edge data: {:?}", e); - return Some(Err(e)); - } - }; - match self.edge_type { - EdgeType::Node => { - if let Ok(node) = self.storage.get_node(self.txn, &item_id) { - return Some(Ok(TraversalVal::Node(node))); - } - } - EdgeType::Vec => { - if let Ok(vector) = self.storage.get_vector(self.txn, &item_id) { - return Some(Ok(TraversalVal::Vector(vector))); - } - } - } - } - Err(e) => { - println!("Error decoding edge data: {:?}", e); - return Some(Err(GraphError::DecodeError(e.to_string()))); - } - } - } - None + self.iter.next().map(|item| { + let start_node = item?; + let nodes = + self.storage + .get_out_nodes(self.txn, self.edge_label, &start_node.id())?; + Ok(TraversalVal::NodeArray(nodes)) + }) } } -pub trait OutAdapter<'a, T>: Iterator> { - /// Returns an iterator containing the nodes that have an outgoing edge with the given label. - /// - /// Note that the `edge_label` cannot be empty and must be a valid, existing edge label. - /// - /// To provide safety, you cannot get all outgoing nodes as it would be ambiguous as to what - /// type that resulting node would be. +pub trait OutAdapter<'a, S: Storage + ?Sized, I: Iterator>>: + Iterator> +{ fn out( self, edge_label: &'a str, - edge_type: &'a EdgeType, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> OutAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> OutAdapter<'a, S, I> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { #[inline] fn out( self, edge_label: &'a str, - edge_type: &'a EdgeType, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let db = Arc::clone(&self.storage); - let storage = Arc::clone(&self.storage); - let txn = self.txn; - - let iter = self - .inner - .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::out_edge_key( - &match item { - Ok(item) => item.id(), - Err(_) => return None, - }, - &edge_label_hash, - ); - match db - .out_edges_db - .lazily_decode_data() - .get_duplicates(txn, &prefix) - { - Ok(Some(iter)) => Some(OutNodesIterator { - iter, - storage: Arc::clone(&db), - edge_type, - txn, - }), - Ok(None) => None, - Err(e) => { - println!("{} Error getting out edges: {:?}", line!(), e); - // return Err(e); - None - } - } - }) - .flatten(); - + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { - inner: iter, - storage, - txn, + inner: OutNodesIterator { + iter: self.inner, + storage: self.storage.clone(), + txn: self.txn, + edge_label, + }, + storage: self.storage, + txn: self.txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/out/out_e.rs b/helixdb/src/helix_engine/graph_core/ops/out/out_e.rs index 1634729a0..eb85342af 100644 --- a/helixdb/src/helix_engine/graph_core/ops/out/out_e.rs +++ b/helixdb/src/helix_engine/graph_core/ops/out/out_e.rs @@ -4,114 +4,65 @@ use crate::{ ops::tr_val::{Traversable, TraversalVal}, traversal_iter::RoTraversalIterator, }, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, - protocol::label_hash::hash_label, + helix_storage::Storage, }; -use heed3::{types::Bytes, RoTxn}; use std::sync::Arc; -pub struct OutEdgesIterator<'a, T> { - pub iter: heed3::RoIter< - 'a, - Bytes, - heed3::types::LazyDecode, - heed3::iteration_method::MoveOnCurrentKeyDuplicates, - >, - pub storage: Arc, - pub txn: &'a T, +pub struct OutEdgesIterator<'a, I, S: Storage + ?Sized> { + iter: I, + storage: Arc, + txn: &'a S::RoTxn<'a>, + edge_label: &'a str, } -impl<'a> Iterator for OutEdgesIterator<'a, RoTxn<'a>> { +impl<'a, I, S> Iterator for OutEdgesIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, +{ type Item = Result; fn next(&mut self) -> Option { - while let Some(Ok((_, data))) = self.iter.next() { - match data.decode() { - Ok(data) => { - let (_, edge_id) = match HelixGraphStorage::unpack_adj_edge_data(&data) { - Ok(data) => data, - Err(e) => { - println!("Error unpacking edge data: {:?}", e); - return Some(Err(e)); - } - }; - if let Ok(edge) = self.storage.get_edge(self.txn, &edge_id) { - return Some(Ok(TraversalVal::Edge(edge))); - } - } - Err(e) => { - println!("Error decoding edge data: {:?}", e); - return Some(Err(GraphError::DecodeError(e.to_string()))); - } - } - } - None + self.iter.next().map(|item| { + let start_node = item?; + let edges = self + .storage + .get_out_edges(self.txn, self.edge_label, &start_node.id())?; + Ok(TraversalVal::EdgeArray(edges)) + }) } } -pub trait OutEdgesAdapter<'a, T>: Iterator> { - /// Returns an iterator containing the edges that have an outgoing edge with the given label. - /// - /// Note that the `edge_label` cannot be empty and must be a valid, existing edge label. - /// - /// To provide safety, you cannot get all outgoing edges as it would be ambiguous as to what - /// type that resulting edge would be. +pub trait OutEdgesAdapter<'a, S: Storage + ?Sized, I: Iterator>>: + Iterator> +{ fn out_e( self, edge_label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator> + 'a> OutEdgesAdapter<'a, RoTxn<'a>> - for RoTraversalIterator<'a, I> +impl<'a, I, S> OutEdgesAdapter<'a, S, I> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { #[inline] fn out_e( self, edge_label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { - // iterate through the iterator and create a new iterator on the out edges - let db = Arc::clone(&self.storage); - let storage = Arc::clone(&self.storage); - let txn = self.txn; - let iter = self - .inner - .filter_map(move |item| { - let edge_label_hash = hash_label(edge_label, None); - match item { - Ok(item) => { - let prefix = HelixGraphStorage::out_edge_key(&item.id(), &edge_label_hash); - match db - .out_edges_db - .lazily_decode_data() - .get_duplicates(txn, &prefix) - { - Ok(Some(iter)) => Some(OutEdgesIterator { - iter, - storage: Arc::clone(&db), - txn, - }), - Ok(None) => None, - Err(e) => { - println!("{} Error getting out edges: {:?}", line!(), e); - // return Err(e); - None - } - } - } - Err(e) => { - println!("{} Error getting oupt edges: {:?}", line!(), e); - None - } - } - }) - .flatten(); + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { - inner: iter, - storage, - txn, + inner: OutEdgesIterator { + iter: self.inner, + storage: self.storage.clone(), + txn: self.txn, + edge_label, + }, + storage: self.storage, + txn: self.txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/source/add_e.rs b/helixdb/src/helix_engine/graph_core/ops/source/add_e.rs index 70c387f2d..0e02133bf 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/add_e.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/add_e.rs @@ -4,15 +4,11 @@ use super::super::tr_val::TraversalVal; use crate::{ helix_engine::{ graph_core::traversal_iter::RwTraversalIterator, - storage_core::storage_core::HelixGraphStorage, types::GraphError, vector_core::hnsw::HNSW, - }, - protocol::{ - items::{v6_uuid, Edge}, - label_hash::hash_label, - value::Value, + types::GraphError, }, + helix_storage::Storage, + protocol::value::Value, }; -use heed3::PutFlags; use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] @@ -43,149 +39,40 @@ impl Iterator for AddE { } } -pub trait AddEAdapter<'a, 'b>: Iterator> { +pub trait AddEAdapter<'a, 'b, S: Storage + ?Sized>: + Iterator> +{ fn add_e( self, label: &'a str, properties: Option>, from_node: u128, to_node: u128, - should_check: bool, - edge_type: EdgeType, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>>; - - fn node_vec_exists(&self, node_vec_id: &u128, edge_type: EdgeType) -> bool; + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S>; } -impl<'a, 'b, I: Iterator>> AddEAdapter<'a, 'b> - for RwTraversalIterator<'a, 'b, I> +impl<'a, 'b, I, S> AddEAdapter<'a, 'b, S> for RwTraversalIterator<'a, 'b, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline(always)] fn add_e( - self, + mut self, label: &'a str, properties: Option>, from_node: u128, to_node: u128, - should_check: bool, - // edge_types: (EdgeType, EdgeType), - edge_type: EdgeType, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>> { - let edge = Edge { - id: v6_uuid(), - label: label.to_string(), - properties: properties.map(|props| props.into_iter().collect()), - from_node, - to_node, - }; - - let mut result: Result = Ok(TraversalVal::Empty); - - // if should_check { - // match edge_types { - // (EdgeType::Node, EdgeType::Node) => { - // if !(self.node_vec_exists(&from_node, EdgeType::Node) - // && self.node_vec_exists(&to_node, EdgeType::Node)) - // { - // result = Err(GraphError::NodeNotFound); - // } - // } - // (EdgeType::Vec, EdgeType::Vec) => { - // if !(self.node_vec_exists(&from_node, EdgeType::Vec) - // && self.node_vec_exists(&to_node, EdgeType::Vec)) - // { - // result = Err(GraphError::NodeNotFound); - // } - // } - // (EdgeType::Node, EdgeType::Vec) => { - // if !(self.node_vec_exists(&from_node, EdgeType::Node) - // && self.node_vec_exists(&to_node, EdgeType::Vec)) - // { - // result = Err(GraphError::NodeNotFound); - // } - // } - // (EdgeType::Vec, EdgeType::Node) => { - // if !(self.node_vec_exists(&from_node, EdgeType::Vec) - // && self.node_vec_exists(&to_node, EdgeType::Node)) - // { - // result = Err(GraphError::NodeNotFound); - // } - // } - // } - // } - - match edge.encode_edge() { - Ok(bytes) => { - if let Err(e) = self.storage.edges_db.put_with_flags( - self.txn, - PutFlags::APPEND, - &HelixGraphStorage::edge_key(&edge.id), - &bytes, - ) { - result = Err(GraphError::from(e)); - } - } - Err(e) => result = Err(GraphError::from(e)), - } - - let label_hash = hash_label(edge.label.as_str(), None); - - match self.storage.out_edges_db.put_with_flags( - self.txn, - PutFlags::APPEND_DUP, - &HelixGraphStorage::out_edge_key(&from_node, &label_hash), - &HelixGraphStorage::pack_edge_data(&to_node, &edge.id), - ) { - Ok(_) => {} - Err(e) => { - println!("add_e => error adding out edge: {:?}", e); - result = Err(GraphError::from(e)); - } - } - - match self.storage.in_edges_db.put_with_flags( - self.txn, - PutFlags::APPEND_DUP, - &HelixGraphStorage::in_edge_key(&to_node, &label_hash), - &HelixGraphStorage::pack_edge_data(&from_node, &edge.id), - ) { - Ok(_) => {} - Err(e) => { - println!("add_e => error adding in edge: {:?}", e); - result = Err(GraphError::from(e)); - } - } - - let result = match result { - Ok(_) => Ok(TraversalVal::Edge(edge)), - Err(_) => Err(GraphError::EdgeNotFound), - }; + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S> { + let result = self + .storage + .add_edge(self.txn, label, properties, from_node, to_node) + .map(TraversalVal::Edge); RwTraversalIterator { - inner: std::iter::once(result), // TODO: change to support adding multiple edges + inner: std::iter::once(result), storage: self.storage, txn: self.txn, } } - - fn node_vec_exists(&self, node_vec_id: &u128, edge_type: EdgeType) -> bool { - let exists = match edge_type { - EdgeType::Node => self - .storage - .nodes_db - .get(self.txn, &HelixGraphStorage::node_key(&node_vec_id)) - .map_or(false, |node| node.is_some()), - EdgeType::Vec => self - .storage - .vectors - .get_vector(self.txn, *node_vec_id, 0, false) - .is_ok(), - }; - - if !exists { - return false; - } - - true - } } diff --git a/helixdb/src/helix_engine/graph_core/ops/source/add_n.rs b/helixdb/src/helix_engine/graph_core/ops/source/add_n.rs index af2f04d12..7ca4d412b 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/add_n.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/add_n.rs @@ -1,17 +1,14 @@ use super::super::tr_val::TraversalVal; use crate::{ helix_engine::{ - bm25::bm25::{BM25Flatten, BM25}, graph_core::traversal_iter::RwTraversalIterator, types::GraphError, }, + helix_storage::Storage, protocol::{ - filterable::Filterable, - items::{v6_uuid, Node}, value::Value, }, }; -use heed3::PutFlags; pub struct AddNIterator { inner: std::iter::Once>, @@ -25,103 +22,38 @@ impl Iterator for AddNIterator { } } -pub trait AddNAdapter<'a, 'b>: Iterator> { +pub trait AddNAdapter<'a, 'b, S: Storage + ?Sized>: Iterator> { fn add_n( self, label: &'a str, properties: Option>, secondary_indices: Option<&'a [&str]>, - ) -> RwTraversalIterator<'a, 'b, std::iter::Once>>; + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S>; } -impl<'a, 'b, I: Iterator>> AddNAdapter<'a, 'b> - for RwTraversalIterator<'a, 'b, I> +impl<'a, 'b, I, S> AddNAdapter<'a, 'b, S> for RwTraversalIterator<'a, 'b, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { fn add_n( - self, + mut self, label: &'a str, properties: Option>, secondary_indices: Option<&'a [&str]>, - ) -> RwTraversalIterator<'a, 'b, std::iter::Once>> { - let node = Node { - id: v6_uuid(), - label: label.to_string(), // TODO: just &str or Cow<'a, str> - properties: properties.map(|props| props.into_iter().collect()), + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S> { + let result = self.storage.add_node( + self.txn, + label, + properties, + secondary_indices, + ); + + let result = match result { + Ok(node) => Ok(TraversalVal::Node(node)), + Err(e) => Err(e), }; - let secondary_indices = secondary_indices.unwrap_or(&[]).to_vec(); - let mut result: Result = Ok(TraversalVal::Empty); - - match node.encode_node() { - Ok(bytes) => { - if let Err(e) = self.storage.nodes_db.put_with_flags( - self.txn, - PutFlags::APPEND, - &node.id, - &bytes, - ) { - result = Err(GraphError::from(e)); - } - } - Err(e) => result = Err(GraphError::from(e)), - } - - for index in secondary_indices { - match self.storage.secondary_indices.get(index) { - Some(db) => { - let key = match node.check_property(&index) { - Ok(value) => value, - Err(e) => { - result = Err(e); - continue; - } - }; - // look into if there is a way to serialize to a slice - match bincode::serialize(&key) { - Ok(serialized) => { - // possibly append dup - - if let Err(e) = db.put(self.txn, &serialized, &node.id) { - println!("{} Error adding node to secondary index: {:?}", line!(), e); - result = Err(GraphError::from(e)); - } - } - Err(e) => result = Err(GraphError::from(e)), - } - } - None => { - result = Err(GraphError::New(format!( - "Secondary Index {} not found", - index - ))); - } - } - } - - // auto inserts to bm25 if should_add_to_bm25 is true - if node.properties.is_some() { - let mut data = node - .properties - .as_ref() - .map(|props| props.flatten_bm25()) - .unwrap_or_default(); - data.push_str(&node.label); - match self.storage.bm25.insert_doc(self.txn, node.id, &data) { - Ok(_) => {} - Err(e) => { - result = Err(GraphError::from(e)); - } - } - } - - if result.is_ok() { - result = Ok(TraversalVal::Node(node.clone())); - } else { - result = Err(GraphError::New(format!( - "Failed to add node to secondary indices" - ))); - } - RwTraversalIterator { inner: std::iter::once(result), storage: self.storage, diff --git a/helixdb/src/helix_engine/graph_core/ops/source/e_from_id.rs b/helixdb/src/helix_engine/graph_core/ops/source/e_from_id.rs index c80b63d54..9ea5ef232 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/e_from_id.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/e_from_id.rs @@ -1,36 +1,35 @@ +use std::sync::Arc; +use std::iter::Once; + use crate::{ helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, + helix_storage::Storage, protocol::items::Edge, }; -use heed3::RoTxn; -use std::{iter::Once, sync::Arc}; -pub struct EFromId<'a, T> { +pub struct EFromId<'a, T, S: Storage + ?Sized> { iter: Once>, - storage: Arc, + storage: Arc, txn: &'a T, id: &'a u128, } -impl<'a> Iterator for EFromId<'a, RoTxn<'a>> { - +impl<'a, S: Storage + ?Sized> Iterator for EFromId<'a, S::RoTxn<'a>, S> { type Item = Result; fn next(&mut self) -> Option { self.iter.next().map(|_| { - let edge: Edge = match self.storage.get_edge(self.txn, self.id) { - Ok(edge) => edge, - Err(e) => return Err(e), - }; + let edge: Edge = self.storage.get_edge(self.txn, self.id)?; Ok(TraversalVal::Edge(edge)) }) } } -pub trait EFromIdAdapter<'a>: Iterator> { +pub trait EFromIdAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ type OutputIter: Iterator>; /// Returns an iterator containing the edge with the given id. @@ -39,10 +38,10 @@ pub trait EFromIdAdapter<'a>: Iterator> fn e_from_id(self, id: &'a u128) -> Self::OutputIter; } -impl<'a, I: Iterator>> EFromIdAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I: Iterator>, S: Storage + ?Sized> + EFromIdAdapter<'a, S> for RoTraversalIterator<'a, I, S> { - type OutputIter = RoTraversalIterator<'a, EFromId<'a, RoTxn<'a>>>; + type OutputIter = RoTraversalIterator<'a, EFromId<'a, S::RoTxn<'a>, S>, S>; #[inline] fn e_from_id(self, id: &'a u128) -> Self::OutputIter { diff --git a/helixdb/src/helix_engine/graph_core/ops/source/e_from_type.rs b/helixdb/src/helix_engine/graph_core/ops/source/e_from_type.rs index 3bf77b5bd..c9075dc13 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/e_from_type.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/e_from_type.rs @@ -3,6 +3,7 @@ use crate::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, types::GraphError, }, + helix_storage::Storage, protocol::items::Edge, }; use heed3::{ @@ -35,26 +36,25 @@ impl<'a> Iterator for EFromType<'a> { None } } -pub trait EFromTypeAdapter<'a>: Iterator> { +pub trait EFromTypeAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn e_from_type( self, label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> EFromTypeAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> EFromTypeAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline] fn e_from_type( self, label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let iter = self - .storage - .edges_db - .lazily_decode_data() - .iter(self.txn) - .unwrap(); + ) -> RoTraversalIterator<'a, impl Iterator>, S> { + let iter = self.storage.get_all_edges(self.txn).unwrap(); RoTraversalIterator { inner: EFromType { iter, label }, storage: self.storage, diff --git a/helixdb/src/helix_engine/graph_core/ops/source/n_from_id.rs b/helixdb/src/helix_engine/graph_core/ops/source/n_from_id.rs index 665800180..5102c0de5 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/n_from_id.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/n_from_id.rs @@ -1,36 +1,36 @@ +use std::sync::Arc; +use std::iter::Once; + use crate::{ helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, + helix_storage::Storage, protocol::items::Node, }; -use heed3::RoTxn; -use std::{iter::Once, sync::Arc}; -pub struct NFromId<'a, T> { +pub struct NFromId<'a, T, S: Storage + ?Sized> { iter: Once>, - storage: Arc, + storage: Arc, txn: &'a T, id: u128, } -impl<'a> Iterator for NFromId<'a, RoTxn<'a>> { +impl<'a, S: Storage + ?Sized> Iterator for NFromId<'a, S::RoTxn<'a>, S> { type Item = Result; fn next(&mut self) -> Option { self.iter.next().map(|_| { - let node: Node = match self.storage.get_node(self.txn, &self.id) { - Ok(node) => node, - Err(e) => return Err(e), - }; + let node: Node = self.storage.get_node(self.txn, &self.id)?; Ok(TraversalVal::Node(node)) }) } } -pub trait NFromIdAdapter<'a>: Iterator> { +pub trait NFromIdAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ type OutputIter: Iterator>; /// Returns an iterator containing the node with the given id. @@ -39,10 +39,10 @@ pub trait NFromIdAdapter<'a>: Iterator> fn n_from_id(self, id: &u128) -> Self::OutputIter; } -impl<'a, I: Iterator>> NFromIdAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I: Iterator>, S: Storage + ?Sized> + NFromIdAdapter<'a, S> for RoTraversalIterator<'a, I, S> { - type OutputIter = RoTraversalIterator<'a, NFromId<'a, RoTxn<'a>>>; + type OutputIter = RoTraversalIterator<'a, NFromId<'a, S::RoTxn<'a>, S>, S>; #[inline] fn n_from_id(self, id: &u128) -> Self::OutputIter { diff --git a/helixdb/src/helix_engine/graph_core/ops/source/n_from_index.rs b/helixdb/src/helix_engine/graph_core/ops/source/n_from_index.rs index 7d8eba609..a40bc968d 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/n_from_index.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/n_from_index.rs @@ -1,98 +1,66 @@ use crate::{ helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, + helix_storage::Storage, protocol::{items::Node, value::Value}, }; -use heed3::{byteorder::BE, types::Bytes, RoTxn}; use serde::Serialize; -use std::{iter::Once, sync::Arc}; +use std::sync::Arc; -pub struct NFromIndex<'a> { - iter: - heed3::RoPrefix<'a, heed3::types::Bytes, heed3::types::LazyDecode>>, - txn: &'a RoTxn<'a>, - storage: Arc, +pub struct NFromIndex { + nodes: Vec, + storage: Arc, } -impl<'a> Iterator for NFromIndex<'a> { +impl Iterator for NFromIndex { type Item = Result; fn next(&mut self) -> Option { - while let Some(value) = self.iter.next() { - let (key_, value) = value.unwrap(); - match value.decode() { - Ok(value) => match self.storage.get_node(self.txn, &value) { - Ok(node) => return Some(Ok(TraversalVal::Node(node))), - Err(e) => { - println!("{} Error getting node: {:?}", line!(), e); - return Some(Err(GraphError::ConversionError(e.to_string()))); - } - }, - - Err(e) => return Some(Err(GraphError::ConversionError(e.to_string()))), - } - } - None + self.nodes.pop().map(|n| Ok(TraversalVal::Node(n))) } } -pub trait NFromIndexAdapter<'a, K: Into + Serialize>: - Iterator> +pub trait NFromIndexAdapter<'a, K, S>: Iterator> +where + K: Into + Serialize, + S: Storage + ?Sized, { - type OutputIter: Iterator>; - - /// Returns a new iterator that will return the node from the secondary index. - /// - /// # Arguments - /// - /// * `index` - The name of the secondary index. - /// * `key` - The key to search for in the secondary index. - /// - /// Note that both the `index` and `key` must be provided. - /// The index must be a valid and existing secondary index and the key should match the type of the index. - fn n_from_index(self, index: &'a str, key: &'a K) -> Self::OutputIter - where - K: Into + Serialize + Clone; + fn n_from_index( + self, + index: &'a str, + key: &'a K, + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>, K: Into + Serialize + 'a> - NFromIndexAdapter<'a, K> for RoTraversalIterator<'a, I> +impl<'a, I, K, S> NFromIndexAdapter<'a, K, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + K: Into + Serialize + 'a, + S: Storage + ?Sized, { - type OutputIter = RoTraversalIterator<'a, NFromIndex<'a>>; - #[inline] - fn n_from_index(self, index: &'a str, key: &'a K) -> Self::OutputIter - where - K: Into + Serialize + Clone, - { - let db = self - .storage - .secondary_indices - .get(index) - .ok_or(GraphError::New(format!( - "Secondary Index {} not found", - index - ))) - .unwrap(); - println!("DB: {:?}", self.storage.secondary_indices); - let res = db - .lazily_decode_data() - .prefix_iter(self.txn, &bincode::serialize(&Value::from(key)).unwrap()) - .unwrap(); + fn n_from_index( + self, + index: &'a str, + key: &'a K, + ) -> RoTraversalIterator<'a, impl Iterator>, S> { + let storage = self.storage.clone(); + let txn = self.txn; + let nodes = storage + .node_from_index(txn, index, key) + .unwrap_or_default(); - let n_from_index = NFromIndex { - iter: res, - txn: self.txn, - storage: Arc::clone(&self.storage), + let iter = NFromIndex { + nodes, + storage: Arc::clone(&storage), }; RoTraversalIterator { - inner: n_from_index, - storage: self.storage, - txn: self.txn, + inner: iter, + storage, + txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/source/n_from_type.rs b/helixdb/src/helix_engine/graph_core/ops/source/n_from_type.rs index b3f685fa1..b1ff19337 100644 --- a/helixdb/src/helix_engine/graph_core/ops/source/n_from_type.rs +++ b/helixdb/src/helix_engine/graph_core/ops/source/n_from_type.rs @@ -3,6 +3,7 @@ use crate::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, types::GraphError, }, + helix_storage::Storage, protocol::items::Node, }; use heed3::{ @@ -38,29 +39,28 @@ impl<'a> Iterator for NFromType<'a> { None } } -pub trait NFromTypeAdapter<'a>: Iterator> { +pub trait NFromTypeAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ /// Returns an iterator containing the nodes with the given label. /// /// Note that the `label` cannot be empty and must be a valid, existing node label. fn n_from_type( self, label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> NFromTypeAdapter<'a> - for RoTraversalIterator<'a, I> -{ +impl<'a, I, S> NFromTypeAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, +{ #[inline] fn n_from_type( self, label: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let iter = self - .storage - .nodes_db - .lazily_decode_data() - .iter(self.txn) - .unwrap(); + ) -> RoTraversalIterator<'a, impl Iterator>, S> { + let iter = self.storage.get_all_nodes(self.txn).unwrap(); RoTraversalIterator { inner: NFromType { iter, label }, storage: self.storage, diff --git a/helixdb/src/helix_engine/graph_core/ops/tr_val.rs b/helixdb/src/helix_engine/graph_core/ops/tr_val.rs index 9484fba6d..5b976e5ab 100644 --- a/helixdb/src/helix_engine/graph_core/ops/tr_val.rs +++ b/helixdb/src/helix_engine/graph_core/ops/tr_val.rs @@ -13,6 +13,8 @@ use std::hash::Hash; pub enum TraversalVal { Node(Node), Edge(Edge), + NodeArray(Vec), + EdgeArray(Vec), Vector(HVector), Count(Count), Path((Vec, Vec)), diff --git a/helixdb/src/helix_engine/graph_core/ops/util/dedup.rs b/helixdb/src/helix_engine/graph_core/ops/util/dedup.rs index d147855fc..a36a8f5ce 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/dedup.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/dedup.rs @@ -7,6 +7,7 @@ use crate::helix_engine::{ }, types::GraphError, }; +use crate::helix_storage::Storage; pub struct Dedup { iter: I, @@ -36,19 +37,21 @@ where } } -pub trait DedupAdapter<'a>: Iterator { +pub trait DedupAdapter<'a, S: Storage + ?Sized>: Iterator { /// Dedup returns an iterator that will return unique items when collected fn dedup( self, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator>> DedupAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> DedupAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { fn dedup( self, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { { let upper_bound = match self.inner.size_hint() { (_, Some(upper_bound)) => upper_bound, diff --git a/helixdb/src/helix_engine/graph_core/ops/util/drop.rs b/helixdb/src/helix_engine/graph_core/ops/util/drop.rs index c0a25d0eb..979514397 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/drop.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/drop.rs @@ -1,42 +1,29 @@ use crate::helix_engine::{ graph_core::ops::tr_val::TraversalVal, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }; -use heed3::RwTxn; +use crate::helix_storage::Storage; use std::sync::Arc; pub struct Drop { pub iter: I, } -impl<'a> Drop>> { - pub fn drop_traversal( - iter: Vec>, - storage: Arc, - txn: &mut RwTxn, +impl Drop +where + I: Iterator>, +{ + pub fn drop_traversal<'a, S: Storage + ?Sized>( + iter: I, + storage: Arc, + txn: &mut S::RwTxn<'a>, ) -> Result<(), GraphError> { - iter.into_iter() - .try_for_each(|item| -> Result<(), GraphError> { - match item { - Ok(item) => match item { - TraversalVal::Node(node) => match storage.drop_node(txn, &node.id) { - Ok(_) => Ok(()), - Err(e) => return Err(e), - }, - TraversalVal::Edge(edge) => match storage.drop_edge(txn, &edge.id) { - Ok(_) => Ok(()), - Err(e) => return Err(e), - }, - _ => { - return Err(GraphError::ConversionError(format!( - "Incorrect Type: {:?}", - item - ))); - } - }, - Err(e) => return Err(e), - } - }) + iter.try_for_each(|item| -> Result<(), GraphError> { + match item? { + TraversalVal::Node(node) => storage.drop_node(txn, &node.id), + TraversalVal::Edge(edge) => storage.drop_edge(txn, &edge.id), + _ => Err(GraphError::WrongTraversalValue), + } + }) } } diff --git a/helixdb/src/helix_engine/graph_core/ops/util/filter_mut.rs b/helixdb/src/helix_engine/graph_core/ops/util/filter_mut.rs index 1be07fa53..25b67634f 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/filter_mut.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/filter_mut.rs @@ -1,30 +1,25 @@ -use heed3::RwTxn; +use crate::helix_engine::{graph_core::ops::tr_val::TraversalVal, types::GraphError}; +use crate::helix_storage::Storage; -use crate::helix_engine::{ - graph_core::ops::tr_val::TraversalVal, - types::GraphError, -}; - -pub struct FilterMut<'a, I, F> { +pub struct FilterMut<'a, 'env, I, F, S: Storage + ?Sized> { iter: I, - txn: &'a mut RwTxn<'a>, + txn: &'a mut S::RwTxn<'env>, f: F, } -impl<'a, I, F> Iterator for FilterMut<'a, I, F> +impl<'a, 'env, I, F, S: Storage + ?Sized> Iterator for FilterMut<'a, 'env, I, F, S> where I: Iterator>, - F: FnMut(&mut I::Item, &mut RwTxn) -> bool, + F: FnMut(&I::Item, &mut S::RwTxn<'env>) -> bool, { type Item = I::Item; fn next(&mut self) -> Option { - match self.iter.next() { - Some(mut item) => match (self.f)(&mut item, &mut self.txn) { - true => Some(item), - false => None, - }, - None => None, + if let Some(item) = self.iter.next() { + if (self.f)(&item, self.txn) { + return Some(item); + } } + None } } \ No newline at end of file diff --git a/helixdb/src/helix_engine/graph_core/ops/util/filter_ref.rs b/helixdb/src/helix_engine/graph_core/ops/util/filter_ref.rs index d0876f0f2..04db251d7 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/filter_ref.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/filter_ref.rs @@ -1,62 +1,59 @@ -use crate::helix_engine::{graph_core::traversal_iter::RoTraversalIterator, types::GraphError}; +use crate::helix_engine::{ + graph_core::traversal_iter::RoTraversalIterator, types::GraphError, +}; +use crate::helix_storage::Storage; use super::super::tr_val::TraversalVal; -use heed3::RoTxn; -pub struct FilterRef<'a, I, F> { +pub struct FilterRef<'a, I, F, S: Storage + ?Sized> { iter: I, - txn: &'a RoTxn<'a>, + txn: &'a S::RoTxn<'a>, f: F, } -impl<'a, I, F> Iterator for FilterRef<'a, I, F> +impl<'a, I, F, S: Storage + ?Sized> Iterator for FilterRef<'a, I, F, S> where I: Iterator>, - F: Fn(&I::Item, &RoTxn) -> Result, + F: Fn(&I::Item, &S::RoTxn<'a>) -> Result, { type Item = I::Item; fn next(&mut self) -> Option { while let Some(item) = self.iter.next() { - match (self.f)(&item, &self.txn) { - Ok(result) => { - if result { - return Some(item); - } - } - Err(e) => { - return Some(Err(e)); - } + match (self.f)(&item, self.txn) { + Ok(true) => return Some(item), + Ok(false) => continue, + Err(e) => return Some(Err(e)), } } None } } -pub trait FilterRefAdapter<'a>: Iterator { - /// FilterRef filters the iterator by taking a reference - /// to each item and a transaction. +pub trait FilterRefAdapter<'a, S: Storage + ?Sized>: Iterator { fn filter_ref( self, f: F, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&Result, &RoTxn) -> Result; + F: Fn(&Result, &S::RoTxn<'a>) -> Result; } -impl<'a, I: Iterator>> FilterRefAdapter<'a> - for RoTraversalIterator<'a, I> -{ +impl<'a, I, S> FilterRefAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, +{ #[inline] fn filter_ref( self, f: F, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&Result, &RoTxn) -> Result, + F: Fn(&Result, &S::RoTxn<'a>) -> Result, { RoTraversalIterator { - inner: FilterRef { + inner: FilterRef:: { iter: self.inner, txn: self.txn, f, diff --git a/helixdb/src/helix_engine/graph_core/ops/util/map.rs b/helixdb/src/helix_engine/graph_core/ops/util/map.rs index 93a090b89..c02270911 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/map.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/map.rs @@ -2,36 +2,38 @@ use crate::helix_engine::{ graph_core::traversal_iter::{RoTraversalIterator, RwTraversalIterator}, types::GraphError, }; +use crate::helix_storage::Storage; use super::super::tr_val::TraversalVal; -use heed3::RoTxn; -pub struct Map<'a, I, F> { +pub struct Map<'a, I, F, S: Storage + ?Sized> { iter: I, - txn: &'a RoTxn<'a>, + txn: &'a S::RoTxn<'a>, f: F, } // implementing iterator for filter ref -impl<'a, I, F> Iterator for Map<'a, I, F> +impl<'a, I, F, S: Storage + ?Sized> Iterator for Map<'a, I, F, S> where I: Iterator>, - F: FnMut(TraversalVal, &RoTxn<'a>) -> Result, + F: FnMut(TraversalVal, &S::RoTxn<'a>) -> Result, { type Item = I::Item; fn next(&mut self) -> Option { - while let Some(item) = self.iter.next() { + if let Some(item) = self.iter.next() { return match item { - Ok(item) => Some((self.f)(item, &self.txn)), - Err(e) => return Some(Err(e)), + Ok(item) => Some((self.f)(item, self.txn)), + Err(e) => Some(Err(e)), }; } None } } -pub trait MapAdapter<'a>: Iterator> { +pub trait MapAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ /// MapTraversal maps the iterator by taking a reference /// to each item and a transaction. /// @@ -49,24 +51,26 @@ pub trait MapAdapter<'a>: Iterator> { fn map_traversal( self, f: F, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: FnMut(TraversalVal, &RoTxn<'a>) -> Result; + F: FnMut(TraversalVal, &S::RoTxn<'a>) -> Result; } -impl<'a, I: Iterator>> MapAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> MapAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline] fn map_traversal( self, f: F, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: FnMut(TraversalVal, &RoTxn<'a>) -> Result, + F: FnMut(TraversalVal, &S::RoTxn<'a>) -> Result, { RoTraversalIterator { - inner: Map { + inner: Map:: { iter: self.inner, txn: self.txn, f, @@ -97,7 +101,9 @@ where None } } -pub trait MapAdapterMut<'scope, 'env>: Iterator> { +pub trait MapAdapterMut<'scope, 'env, S: Storage + ?Sized>: + Iterator> +{ /// MapTraversalMut maps the iterator by taking a mutable /// reference to each item and a transaction. /// @@ -107,19 +113,21 @@ pub trait MapAdapterMut<'scope, 'env>: Iterator( self, f: F, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>> + ) -> RwTraversalIterator<'scope, 'env, impl Iterator>, S> where F: Fn(Result) -> Result; } -impl<'scope, 'env, I: Iterator>> MapAdapterMut<'scope, 'env> - for RwTraversalIterator<'scope, 'env, I> +impl<'scope, 'env, I, S> MapAdapterMut<'scope, 'env, S> for RwTraversalIterator<'scope, 'env, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { #[inline] fn map_traversal_mut( self, f: F, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>> + ) -> RwTraversalIterator<'scope, 'env, impl Iterator>, S> where F: Fn(I::Item) -> Result, { diff --git a/helixdb/src/helix_engine/graph_core/ops/util/paths.rs b/helixdb/src/helix_engine/graph_core/ops/util/paths.rs index 661f2687e..607c5ad8a 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/paths.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/paths.rs @@ -1,145 +1,71 @@ use crate::{ helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, - protocol::{items::Edge, label_hash::hash_label}, -}; -use heed3::RoTxn; -use std::{ - collections::{HashMap, HashSet, VecDeque}, - sync::Arc, + helix_storage::Storage, }; +use std::sync::Arc; #[derive(Debug, Clone)] -pub enum PathType { - From(u128), - To(u128), +pub enum PathType<'a> { + From(&'a u128), + To(&'a u128), } -pub struct ShortestPathIterator<'a, I> { +pub struct ShortestPathIterator<'a, I, S: Storage + ?Sized> { iter: I, - path_type: PathType, + path_type: PathType<'a>, edge_label: Option<&'a str>, - storage: Arc, - txn: &'a RoTxn<'a>, + storage: Arc, + txn: &'a S::RoTxn<'a>, } -impl<'a, I: Iterator>> Iterator - for ShortestPathIterator<'a, I> +impl<'a, I, S> Iterator for ShortestPathIterator<'a, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { type Item = Result; - /// Returns the next outgoing node by decoding the edge id and then getting the edge and node fn next(&mut self) -> Option { - match self.iter.next() { - Some(Ok(TraversalVal::Node(node))) => { - let (from, to) = match self.path_type { - PathType::From(from) => (from, node.id), - PathType::To(to) => (node.id, to), - }; - - let mut queue = VecDeque::with_capacity(32); - let mut visited = HashSet::with_capacity(64); - let mut parent: HashMap = HashMap::with_capacity(32); - queue.push_back(from); - visited.insert(from); - - let reconstruct_path = |parent: &HashMap, - start_id: &u128, - end_id: &u128| - -> Result { - let mut nodes = Vec::with_capacity(parent.len()); - let mut edges = Vec::with_capacity(parent.len() - 1); - - let mut current = end_id; - - while current != start_id { - nodes.push(self.storage.get_node(self.txn, current)?); - - let (prev_node, edge) = &parent[current]; - edges.push(edge.clone()); - current = prev_node; - } - - nodes.push(self.storage.get_node(self.txn, start_id)?); - - nodes.reverse(); - edges.reverse(); - - Ok(TraversalVal::Path((nodes, edges))) - }; - - while let Some(current_id) = queue.pop_front() { - let out_prefix = self.edge_label.map_or_else( - || current_id.to_be_bytes().to_vec(), - |label| { - HelixGraphStorage::out_edge_key(¤t_id, &hash_label(label, None)) - .to_vec() - }, - ); - - let iter = self - .storage - .out_edges_db - .prefix_iter(self.txn, &out_prefix) - .unwrap(); - - for result in iter { - let (_, value) = result.unwrap(); // TODO: handle error - let (to_node, edge_id) = - HelixGraphStorage::unpack_adj_edge_data(value).unwrap(); // TODO: handle error - - if !visited.contains(&to_node) { - visited.insert(to_node); - let edge = self.storage.get_edge(self.txn, &edge_id).unwrap(); // TODO: handle error - parent.insert(to_node, (current_id, edge)); - - if to_node == to { - return Some(reconstruct_path(&parent, &from, &to)); - } - - queue.push_back(to_node); - } - } - } - Some(Err(GraphError::ShortestPathNotFound)) - } - Some(other) => Some(other), - None => None, - } + self.iter.next().map(|item| { + let start_node = item?; + let (from_id, to_id) = match (self.path_type, start_node) { + (PathType::From(from_id), TraversalVal::Node(to_node)) => (from_id, &to_node.id), + (PathType::To(to_id), TraversalVal::Node(from_node)) => (&from_node.id, to_id), + _ => return Err(GraphError::WrongTraversalValue), + }; + + self.storage + .shortest_path( + self.txn, + self.edge_label.unwrap_or(""), + from_id, + to_id, + ) + .map(TraversalVal::Path) + }) } } -pub trait ShortestPathAdapter<'a, I>: Iterator> { - /// ShortestPath finds the shortest path between two nodes - /// - /// # Arguments - /// - /// * `edge_label` - The label of the edge to use - /// * `from` - The starting node - /// * `to` - The ending node - /// - /// # Example - /// - /// ```rust - /// let node1 = Node { id: 1, label: "Person".to_string(), properties: None }; - /// let node2 = Node { id: 2, label: "Person".to_string(), properties: None }; - /// let traversal = G::new(storage, &txn).shortest_path(Some("knows"), Some(&node1.id), Some(&node2.id)); - /// ``` +pub trait ShortestPathAdapter<'a, S: Storage + ?Sized, I: Iterator>>: + Iterator> +{ fn shortest_path( self, edge_label: Option<&'a str>, from: Option<&'a u128>, to: Option<&'a u128>, - ) -> RoTraversalIterator<'a, ShortestPathIterator<'a, I>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where I: 'a; } -impl<'a, I: Iterator> + 'a> ShortestPathAdapter<'a, I> - for RoTraversalIterator<'a, I> +impl<'a, I, S> ShortestPathAdapter<'a, S, I> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { #[inline] fn shortest_path( @@ -147,26 +73,23 @@ impl<'a, I: Iterator> + 'a> ShortestPath edge_label: Option<&'a str>, from: Option<&'a u128>, to: Option<&'a u128>, - ) -> RoTraversalIterator<'a, ShortestPathIterator<'a, I>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where I: 'a, { - let storage = Arc::clone(&self.storage); - let txn = self.txn; - RoTraversalIterator { inner: ShortestPathIterator { iter: self.inner, path_type: match (from, to) { - (Some(from), None) => PathType::From(*from), - (None, Some(to)) => PathType::To(*to), - _ => panic!("Invalid shortest path"), + (Some(from), None) => PathType::From(from), + (None, Some(to)) => PathType::To(to), + _ => panic!("Invalid shortest path: must provide either from or to"), }, edge_label, - storage, - txn, + storage: self.storage.clone(), + txn: self.txn, }, - storage: Arc::clone(&self.storage), + storage: self.storage, txn: self.txn, } } diff --git a/helixdb/src/helix_engine/graph_core/ops/util/props.rs b/helixdb/src/helix_engine/graph_core/ops/util/props.rs index c1ad7c974..f36477d07 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/props.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/props.rs @@ -5,6 +5,7 @@ use crate::helix_engine::{ }, types::GraphError, }; +use crate::helix_storage::Storage; pub struct PropsIterator<'a, I> { iter: I, @@ -54,22 +55,25 @@ where } } } -pub trait PropsAdapter<'a, I>: Iterator> { +pub trait PropsAdapter<'a, I, S: Storage + ?Sized>: + Iterator> +{ fn check_property( self, prop: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I> PropsAdapter<'a, I> for RoTraversalIterator<'a, I> +impl<'a, I, S> PropsAdapter<'a, I, S> for RoTraversalIterator<'a, I, S> where I: Iterator>, + S: Storage + ?Sized, { #[inline] fn check_property( self, prop: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { inner: PropsIterator { iter: self.inner, @@ -81,23 +85,24 @@ where } } -impl<'a, 'b, I> PropsAdapter<'a, I> for RwTraversalIterator<'a, 'b, I> +impl<'a, 'b, I, S> PropsAdapter<'a, I, S> for RwTraversalIterator<'a, 'b, I, S> where I: Iterator>, + S: Storage + ?Sized, 'b: 'a, { #[inline] fn check_property( self, prop: &'a str, - ) -> RoTraversalIterator<'a, impl Iterator>> { + ) -> RoTraversalIterator<'a, impl Iterator>, S> { RoTraversalIterator { inner: PropsIterator { iter: self.inner, prop, }, storage: self.storage, - txn: self.txn, + txn: &*self.txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/util/range.rs b/helixdb/src/helix_engine/graph_core/ops/util/range.rs index a776f81a6..83ccca559 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/range.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/range.rs @@ -4,6 +4,7 @@ use crate::helix_engine::{ graph_core::{ops::tr_val::TraversalVal, traversal_iter::RoTraversalIterator}, types::GraphError, }; +use crate::helix_storage::Storage; pub struct Range { iter: I, @@ -44,7 +45,7 @@ where } } -pub trait RangeAdapter<'a>: Iterator { +pub trait RangeAdapter<'a, S: Storage + ?Sized>: Iterator { /// Range returns a slice of the current step between two points /// /// # Arguments @@ -61,21 +62,23 @@ pub trait RangeAdapter<'a>: Iterator { self, start: usize, end: usize, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where Self: Sized + Iterator, Self::Item: Send; } -impl<'a, I: Iterator> + 'a> RangeAdapter<'a> - for RoTraversalIterator<'a, I> -{ +impl<'a, I, S> RangeAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, +{ #[inline(always)] fn range( self, start: usize, end: usize, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where Self: Sized + Iterator, Self::Item: Send, diff --git a/helixdb/src/helix_engine/graph_core/ops/util/update.rs b/helixdb/src/helix_engine/graph_core/ops/util/update.rs index 206651f3f..13d72c27c 100644 --- a/helixdb/src/helix_engine/graph_core/ops/util/update.rs +++ b/helixdb/src/helix_engine/graph_core/ops/util/update.rs @@ -3,9 +3,9 @@ use std::collections::HashMap; use crate::{ helix_engine::{ graph_core::traversal_iter::RwTraversalIterator, - storage_core::{storage_core::HelixGraphStorage, storage_methods::StorageMethods}, types::GraphError, }, + helix_storage::Storage, protocol::value::Value, }; @@ -26,20 +26,22 @@ where } } -pub trait UpdateAdapter<'scope, 'env>: Iterator { +pub trait UpdateAdapter<'scope, 'env, S: Storage + ?Sized>: Iterator { fn update( self, props: Option>, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>>; + ) -> RwTraversalIterator<'scope, 'env, impl Iterator>, S>; } -impl<'scope, 'env, I: Iterator>> UpdateAdapter<'scope, 'env> - for RwTraversalIterator<'scope, 'env, I> +impl<'scope, 'env, I, S> UpdateAdapter<'scope, 'env, S> for RwTraversalIterator<'scope, 'env, I, S> +where + I: Iterator>, + S: Storage + ?Sized, { fn update( - self, + mut self, props: Option>, - ) -> RwTraversalIterator<'scope, 'env, impl Iterator>> + ) -> RwTraversalIterator<'scope, 'env, impl Iterator>, S> { let storage = self.storage.clone(); @@ -51,98 +53,19 @@ impl<'scope, 'env, I: Iterator>> UpdateA for item in self.inner { match item { - Ok(TraversalVal::Node(node)) => match storage.get_node(self.txn, &node.id) { - Ok(mut old_node) => { - if let Some(mut properties) = old_node.properties { - if let Some(ref props) = props { - for (k, v) in props.iter() { - properties.insert(k.clone(), v.clone()); - } - } - for (key, v) in properties.iter() { - if let Some(db) = storage.secondary_indices.get(key) { - match bincode::serialize(v) { - Ok(serialized) => { - if let Err(e) = db.put(self.txn, &serialized, &node.id) - { - vec.push(Err(GraphError::from(e))); - } - } - Err(e) => vec.push(Err(GraphError::from(e))), - } - } - } - old_node.properties = Some(properties); - } else { - let mut properties = HashMap::new(); - if let Some(ref props) = props { - for (k, v) in props.iter() { - properties.insert(k.clone(), v.clone()); - } - } - for (key, v) in properties.iter() { - if let Some(db) = storage.secondary_indices.get(key) { - match bincode::serialize(v) { - Ok(serialized) => { - if let Err(e) = db.put(self.txn, &serialized, &node.id) - { - vec.push(Err(GraphError::from(e))); - } - } - Err(e) => vec.push(Err(GraphError::from(e))), - } - } - } - if properties.len() > 0 { - old_node.properties = Some(properties); - } else { - old_node.properties = None; - } - } - match old_node.encode_node() { - Ok(serialized) => { - match storage.nodes_db.put( - self.txn, - &HelixGraphStorage::node_key(&node.id), - &serialized, - ) { - Ok(_) => vec.push(Ok(TraversalVal::Node(old_node))), - Err(e) => vec.push(Err(GraphError::from(e))), - } - } - Err(e) => vec.push(Err(GraphError::from(e))), - } + Ok(TraversalVal::Node(node)) => { + match storage.update_node(self.txn, &node.id, props.as_ref()) { + Ok(updated_node) => vec.push(Ok(TraversalVal::Node(updated_node))), + Err(e) => vec.push(Err(e)), } - Err(e) => vec.push(Err(e)), - }, - Ok(TraversalVal::Edge(edge)) => match storage.get_edge(self.txn, &edge.id) { - Ok(old_edge) => { - let mut old_edge = old_edge.clone(); - if let Some(mut properties) = old_edge.properties.clone() { - if let Some(ref props) = props { - for (k, v) in props.iter() { - properties.insert(k.clone(), v.clone()); - } - old_edge.properties = Some(properties); - } - } - match old_edge.encode_edge() { - Ok(serialized) => { - match storage.nodes_db.put( - self.txn, - &HelixGraphStorage::edge_key(&edge.id), - &serialized, - ) { - Ok(_) => vec.push(Ok(TraversalVal::Edge(old_edge))), - Err(e) => vec.push(Err(GraphError::from(e))), - } - } - Err(e) => vec.push(Err(GraphError::from(e))), - } + } + Ok(TraversalVal::Edge(edge)) => { + match storage.update_edge(self.txn, &edge.id, props.as_ref()) { + Ok(updated_edge) => vec.push(Ok(TraversalVal::Edge(updated_edge))), + Err(e) => vec.push(Err(e)), } - Err(e) => vec.push(Err(e)), - }, - _ => vec.push(Err(GraphError::New("Unsupported value type".to_string()))), + } + _ => vec.push(Err(GraphError::new("Unsupported value type".to_string()))), } } RwTraversalIterator { diff --git a/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs b/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs index 18442c230..a84b5d8dc 100644 --- a/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs +++ b/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs @@ -10,6 +10,7 @@ use crate::helix_engine::{ }, }; use std::{collections::BinaryHeap, iter::once}; +use crate::helix_storage::Storage; pub struct BruteForceSearchV>> { iter: I, @@ -24,36 +25,42 @@ impl>> Iterator for BruteFor } } -pub trait BruteForceSearchVAdapter<'a>: Iterator> { +pub trait BruteForceSearchVAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn brute_force_search_v( self, query: &Vec, k: usize, - ) -> RoTraversalIterator<'a, impl Iterator>>; + ) -> RoTraversalIterator<'a, impl Iterator>, S>; } -impl<'a, I: Iterator> + 'a> BruteForceSearchVAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> BruteForceSearchVAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage + ?Sized, { fn brute_force_search_v( self, query: &Vec, k: usize, - ) -> RoTraversalIterator<'a, impl Iterator>> { - let mut iter = self.inner.collect::>(); - iter.sort_by(|v1, v2| match (v1, v2) { - (Ok(TraversalVal::Vector(v1)), Ok(TraversalVal::Vector(v2))) => { - let d1 = cosine_similarity(&v1.get_data(), query).unwrap(); - let d2 = cosine_similarity(&v2.get_data(), query).unwrap(); - d1.partial_cmp(&d2).unwrap() + ) -> RoTraversalIterator<'a, impl Iterator>, S> { + let mut iter_vec: Vec<_> = self.inner.collect(); + + iter_vec.sort_by(|v1, v2| { + if let (Ok(TraversalVal::Vector(v1)), Ok(TraversalVal::Vector(v2))) = (v1, v2) { + let d1 = cosine_similarity(v1.get_data(), query).unwrap_or(0.0); + let d2 = cosine_similarity(v2.get_data(), query).unwrap_or(0.0); + d2.partial_cmp(&d1).unwrap_or(std::cmp::Ordering::Equal) + } else { + std::cmp::Ordering::Equal } - _ => panic!("expected vector traversal values"), }); - let iter = iter.into_iter().take(k); + let iter = iter_vec.into_iter().take(k); RoTraversalIterator { - inner: iter.into_iter(), + inner: iter, storage: self.storage, txn: self.txn, } diff --git a/helixdb/src/helix_engine/graph_core/ops/vectors/insert.rs b/helixdb/src/helix_engine/graph_core/ops/vectors/insert.rs index adf7aef2e..a837f2153 100644 --- a/helixdb/src/helix_engine/graph_core/ops/vectors/insert.rs +++ b/helixdb/src/helix_engine/graph_core/ops/vectors/insert.rs @@ -1,70 +1,55 @@ -use heed3::RoTxn; - use super::super::tr_val::TraversalVal; use crate::{ helix_engine::{ graph_core::traversal_iter::RwTraversalIterator, - types::GraphError, + types::{GraphError, VectorError}, vector_core::{hnsw::HNSW, vector::HVector}, }, + helix_storage::{lmdb_storage::LmdbStorage, Storage}, protocol::value::Value, }; use std::sync::Arc; -pub struct InsertVIterator { - inner: std::iter::Once>, -} - -impl Iterator for InsertVIterator { - type Item = Result; - - fn next(&mut self) -> Option { - self.inner.next() - } -} - -pub trait InsertVAdapter<'a, 'b>: Iterator> { - fn insert_v( +pub trait InsertVAdapter<'a, 'b, S: Storage + ?Sized>: + Iterator> +{ + fn insert_v( self, vec: &Vec, - label: &str, fields: Option>, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>> - where - F: Fn(&HVector, &RoTxn) -> bool; + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S>; - fn insert_vs( + fn insert_vs( self, vecs: &Vec>, fields: Option>, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>> - where - F: Fn(&HVector, &RoTxn) -> bool; + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S>; } -impl<'a, 'b, I: Iterator>> InsertVAdapter<'a, 'b> - for RwTraversalIterator<'a, 'b, I> +impl<'a, 'b, I, S> InsertVAdapter<'a, 'b, S> for RwTraversalIterator<'a, 'b, I, S> +where + I: Iterator>, + S: Storage = crate::helix_storage::lmdb_storage::LmdbRwTxn<'b>> + 'static, { - fn insert_v( - self, + fn insert_v( + mut self, query: &Vec, - label: &str, fields: Option>, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>> - where - F: Fn(&HVector, &RoTxn) -> bool, - { - let vector = self - .storage - .vectors - .insert::(self.txn, &query, fields); - - let result = match vector { - Ok(vector) => Ok(TraversalVal::Vector(vector)), - Err(e) => Err(GraphError::from(e)), + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S> { + let result = if let Some(lmdb_storage) = + (self.storage.as_ref() as &dyn std::any::Any).downcast_ref::() + { + lmdb_storage + .vectors + .insert(self.txn, &query, fields) + .map(TraversalVal::Vector) + .map_err(GraphError::from) + } else { + Err(GraphError::from(VectorError::VectorCoreError( + "Vector insert is only supported on LmdbStorage".to_string(), + ))) }; - RwTraversalIterator { inner: std::iter::once(result), storage: self.storage, @@ -72,31 +57,34 @@ impl<'a, 'b, I: Iterator>> InsertVAdapte } } - fn insert_vs( - self, + fn insert_vs( + mut self, vecs: &Vec>, fields: Option>, - ) -> RwTraversalIterator<'a, 'b, impl Iterator>> - where - F: Fn(&HVector, &RoTxn) -> bool, - { - let txn = self.txn; - let storage = Arc::clone(&self.storage); - let iter = vecs - .iter() - .map(|vec| { - let vector = storage.vectors.insert::(txn, &vec, fields.clone()); // TODO: remove clone - match vector { - Ok(vector) => Ok(TraversalVal::Vector(vector)), - Err(e) => Err(GraphError::from(e)), - } - }) - .collect::>(); + ) -> RwTraversalIterator<'a, 'b, impl Iterator>, S> { + let mut results = Vec::with_capacity(vecs.len()); + + if let Some(lmdb_storage) = + (self.storage.as_ref() as &dyn std::any::Any).downcast_ref::() + { + for vec in vecs { + let result = lmdb_storage + .vectors + .insert(self.txn, vec, fields.clone()) + .map(TraversalVal::Vector) + .map_err(GraphError::from); + results.push(result); + } + } else { + results.push(Err(GraphError::from(VectorError::VectorCoreError( + "Vector insert is only supported on LmdbStorage".to_string(), + )))); + } RwTraversalIterator { - inner: iter.into_iter(), + inner: results.into_iter(), storage: self.storage, - txn, + txn: self.txn, } } } diff --git a/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs b/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs index 48ab94ff2..2926d80e9 100644 --- a/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs +++ b/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs @@ -1,18 +1,17 @@ -use heed3::RoTxn; - use super::super::tr_val::TraversalVal; use crate::helix_engine::{ graph_core::traversal_iter::RoTraversalIterator, types::{GraphError, VectorError}, vector_core::{hnsw::HNSW, vector::HVector}, }; +use crate::helix_storage::{lmdb_storage::LmdbStorage, Storage}; +use heed3::RoTxn; use std::iter::once; pub struct SearchV>> { iter: I, } -// implementing iterator for OutIterator impl>> Iterator for SearchV { type Item = Result; @@ -21,67 +20,54 @@ impl>> Iterator for SearchV< } } -pub trait SearchVAdapter<'a>: Iterator> { +pub trait SearchVAdapter<'a, S: Storage + ?Sized>: + Iterator> +{ fn search_v( self, query: &Vec, k: usize, filter: Option<&[F]>, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&HVector, &RoTxn) -> bool; + F: Fn(&HVector, &heed3::RoTxn<'a>) -> bool; } -impl<'a, I: Iterator> + 'a> SearchVAdapter<'a> - for RoTraversalIterator<'a, I> +impl<'a, I, S> SearchVAdapter<'a, S> for RoTraversalIterator<'a, I, S> +where + I: Iterator> + 'a, + S: Storage = crate::helix_storage::lmdb_storage::LmdbRoTxn<'a>> + 'static, { fn search_v( self, query: &Vec, k: usize, filter: Option<&[F]>, - ) -> RoTraversalIterator<'a, impl Iterator>> + ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&HVector, &RoTxn) -> bool, + F: Fn(&HVector, &heed3::RoTxn<'a>) -> bool, { - let vectors = self - .storage - .vectors - .search(self.txn, &query, k, filter, false); + let vectors = if let Some(lmdb_storage) = + (self.storage.as_ref() as &dyn std::any::Any).downcast_ref::() + { + lmdb_storage + .vectors + .search(self.txn, &query, k, filter, false) + } else { + Err(VectorError::VectorCoreError( + "Vector search is only supported on LmdbStorage".to_string(), + )) + }; let iter = match vectors { Ok(vectors) => vectors .into_iter() - .map(|vector| Ok::(TraversalVal::Vector(vector))) + .map(|vector| Ok(TraversalVal::Vector(vector))) + .collect::>() + .into_iter(), + Err(e) => once(Err(GraphError::from(e))) .collect::>() .into_iter(), - Err(VectorError::VectorNotFound(id)) => { - let error = GraphError::VectorError(format!("vector not found for id {}", id)); - once(Err(error)).collect::>().into_iter() - } - Err(VectorError::InvalidVectorData) => { - let error = GraphError::VectorError("invalid vector data".to_string()); - once(Err(error)).collect::>().into_iter() - } - Err(VectorError::EntryPointNotFound) => { - let error = - GraphError::VectorError("no entry point found for hnsw index".to_string()); - once(Err(error)).collect::>().into_iter() - } - Err(VectorError::ConversionError(e)) => { - let error = GraphError::VectorError(format!("conversion error: {}", e)); - once(Err(error)).collect::>().into_iter() - } - Err(VectorError::VectorCoreError(e)) => { - let error = GraphError::VectorError(format!("vector core error: {}", e)); - once(Err(error)).collect::>().into_iter() - } - Err(VectorError::InvalidVectorLength) => { - let error = GraphError::VectorError("invalid vector dimensions!".to_string()); - once(Err(error)).collect::>().into_iter() - } - .collect::>() - .into_iter(), }; let iter = SearchV { iter }; diff --git a/helixdb/src/helix_engine/graph_core/traversal_iter.rs b/helixdb/src/helix_engine/graph_core/traversal_iter.rs index bea919772..c68deacc3 100644 --- a/helixdb/src/helix_engine/graph_core/traversal_iter.rs +++ b/helixdb/src/helix_engine/graph_core/traversal_iter.rs @@ -1,19 +1,18 @@ use std::sync::Arc; -use heed3::{RoTxn, RwTxn, WithTls}; - use super::ops::tr_val::TraversalVal; -use crate::helix_engine::{storage_core::storage_core::HelixGraphStorage, types::GraphError}; +use crate::helix_storage::{lmdb_storage::LmdbStorage, DbRoTxn, DbRwTxn, Storage}; +use crate::helix_engine::types::GraphError; use itertools::Itertools; -pub struct RoTraversalIterator<'a, I> { +pub struct RoTraversalIterator<'a, I, S: Storage + ?Sized> { pub inner: I, - pub storage: Arc, - pub txn: &'a RoTxn<'a>, + pub storage: Arc, + pub txn: &'a S::RoTxn<'a>, } // implementing iterator for TraversalIterator -impl<'a, I> Iterator for RoTraversalIterator<'a, I> +impl<'a, I, S: Storage + ?Sized> Iterator for RoTraversalIterator<'a, I, S> where I: Iterator>, { @@ -24,7 +23,9 @@ where } } -impl<'a, I: Iterator>> RoTraversalIterator<'a, I> { +impl<'a, I: Iterator>, S: Storage + ?Sized> + RoTraversalIterator<'a, I, S> +{ pub fn take_and_collect_to>(self, n: usize) -> B { self.inner .filter_map(|item| item.ok()) @@ -47,14 +48,14 @@ impl<'a, I: Iterator>> RoTraversalIterat self.inner.filter_map(|item| item.ok()).take(1).next() } } -pub struct RwTraversalIterator<'scope, 'env, I> { +pub struct RwTraversalIterator<'scope, 'env, I, S: Storage + ?Sized> { pub inner: I, - pub storage: Arc, - pub txn: &'scope mut RwTxn<'env>, + pub storage: Arc, + pub txn: &'scope mut S::RwTxn<'env>, } // implementing iterator for TraversalIterator -impl<'scope, 'env, I> Iterator for RwTraversalIterator<'scope, 'env, I> +impl<'scope, 'env, I, S: Storage + ?Sized> Iterator for RwTraversalIterator<'scope, 'env, I, S> where I: Iterator>, { @@ -64,8 +65,8 @@ where self.inner.next() } } -impl<'scope, 'env, I: Iterator> RwTraversalIterator<'scope, 'env, I> { - pub fn new(storage: Arc, txn: &'scope mut RwTxn<'env>, inner: I) -> Self { +impl<'scope, 'env, I: Iterator, S: Storage + ?Sized> RwTraversalIterator<'scope, 'env, I, S> { + pub fn new(storage: Arc, txn: &'scope mut S::RwTxn<'env>, inner: I) -> Self { Self { inner, storage, diff --git a/helixdb/src/helix_engine/mod.rs b/helixdb/src/helix_engine/mod.rs index cef55a641..3dad6e3c3 100644 --- a/helixdb/src/helix_engine/mod.rs +++ b/helixdb/src/helix_engine/mod.rs @@ -1,6 +1,5 @@ pub mod bm25; pub mod graph_core; pub mod macros; -pub mod storage_core; pub mod types; pub mod vector_core; diff --git a/helixdb/src/helix_engine/storage_core/storage_core.rs b/helixdb/src/helix_engine/storage_core/storage_core.rs deleted file mode 100644 index 1b66f3cbf..000000000 --- a/helixdb/src/helix_engine/storage_core/storage_core.rs +++ /dev/null @@ -1,440 +0,0 @@ -use crate::{ - helix_engine::{ - bm25::bm25::{HBM25Config, BM25}, - graph_core::config::Config, - storage_core::storage_methods::StorageMethods, - types::GraphError, - vector_core::{ - hnsw::HNSW, - vector::HVector, - vector_core::{HNSWConfig, VectorCore}, - }, - }, - protocol::{ - filterable::Filterable, - items::{Edge, Node}, - label_hash::hash_label, - value::Value, - }, -}; - -use heed3::byteorder::BE; -use heed3::{types::*, Database, DatabaseFlags, Env, EnvOpenOptions, RoTxn, RwTxn, WithTls}; -use std::collections::HashMap; -use std::fs; -use std::path::Path; - -use super::storage_methods::{BasicStorageMethods, DBMethods}; - -// Database names for different stores -const DB_NODES: &str = "nodes"; // For node data (n:) -const DB_EDGES: &str = "edges"; // For edge data (e:) -const DB_OUT_EDGES: &str = "out_edges"; // For outgoing edge indices (o:) -const DB_IN_EDGES: &str = "in_edges"; // For incoming edge indices (i:) - -// Key prefixes for different types of data - -pub struct HelixGraphStorage { - // TODO: maybe make not public? - pub graph_env: Env, - pub nodes_db: Database, Bytes>, - pub edges_db: Database, Bytes>, - pub out_edges_db: Database, - pub in_edges_db: Database, - pub secondary_indices: HashMap>>, - pub vectors: VectorCore, - pub bm25: HBM25Config, -} - -impl HelixGraphStorage { - pub fn new(path: &str, config: Config) -> Result { - fs::create_dir_all(path)?; - - let db_size = if config.db_max_size_gb.unwrap_or(100) >= 9999 { - 9998 - } else { - config.db_max_size_gb.unwrap_or(100) - }; - - // Configure and open LMDB environment - let graph_env = unsafe { - EnvOpenOptions::new() - .map_size(db_size * 1024 * 1024 * 1024) // GB - .max_dbs(20) - .max_readers(200) - // .flags(EnvFlags::NO_META_SYNC) - // .flags(EnvFlags::MAP_ASYNC) - // .flags(EnvFlags::NO_SYNC) - .open(Path::new(path))? - }; - - let mut wtxn = graph_env.write_txn()?; - - // Create/open all necessary databases - let nodes_db = graph_env - .database_options() - .types::, Bytes>() - .name(DB_NODES) - .create(&mut wtxn)?; - let edges_db = graph_env - .database_options() - .types::, Bytes>() - .name(DB_EDGES) - .create(&mut wtxn)?; - let out_edges_db: Database = graph_env - .database_options() - .types::() - .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) // TODO: remove as well? - .name(DB_OUT_EDGES) - .create(&mut wtxn)?; - let in_edges_db: Database = graph_env - .database_options() - .types::() - .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) // TODO: remove as well? - .name(DB_IN_EDGES) - .create(&mut wtxn)?; - - // Create secondary indices - let mut secondary_indices = HashMap::new(); - if let Some(indexes) = config.graph_config.secondary_indices { - for index in indexes { - secondary_indices.insert( - index.clone(), - graph_env - .database_options() - .types::>() - .flags(DatabaseFlags::DUP_SORT) - .name(&index) - .create(&mut wtxn)?, - ); - } - } - - let vectors = VectorCore::new( - &graph_env, - &mut wtxn, - HNSWConfig::new( - config.vector_config.m, - config.vector_config.ef_construction, - config.vector_config.ef_search, - ), - )?; - let bm25 = HBM25Config::new(&graph_env, &mut wtxn)?; - - wtxn.commit()?; - Ok(Self { - graph_env, - nodes_db, - edges_db, - out_edges_db, - in_edges_db, - secondary_indices, - vectors, - bm25, - }) - } - - pub fn get_random_node(&self, txn: &RoTxn) -> Result { - match self.nodes_db.first(&txn)? { - Some((_, data)) => Ok(bincode::deserialize(data)?), - None => Err(GraphError::NodeNotFound), - } - } - - // todo look into using a shorter hash for space efficiency - // #[inline(always)] - // pub fn hash_label(label: &str) -> [u8; 4] { - // let mut hash = twox_hash::XxHash32::with_seed(0); - // hash.write(label.as_bytes()); - // hash.finish_32().to_le_bytes() - // } - - // #[inline(always)] - // pub fn node_key(id: &u128) -> [u8; 16] { - // id.to_be_bytes() - // } - - // #[inline(always)] - // pub fn edge_key(id: &u128) -> [u8; 16] { - // id.to_be_bytes() - // } - - #[inline(always)] - pub fn node_key(id: &u128) -> &u128 { - id - } - - #[inline(always)] - pub fn edge_key(id: &u128) -> &u128 { - id - } - - #[inline(always)] - pub fn node_label_key(label: &[u8; 4], id: &u128) -> [u8; 20] { - let mut key = [0u8; 20]; - key[0..4].copy_from_slice(label); - key[4..20].copy_from_slice(&id.to_be_bytes()); - key - } - - #[inline(always)] - pub fn edge_label_key(label: &[u8; 4], id: &u128) -> [u8; 20] { - let mut key = [0u8; 20]; - key[0..4].copy_from_slice(label); - key[4..20].copy_from_slice(&id.to_be_bytes()); - key - } - - // key = from-node(16) | label-id(4) ← 20 B - // val = to-node(16) | edge-id(16) ← 32 B (DUPFIXED) - #[inline(always)] - pub fn out_edge_key(from_node_id: &u128, label: &[u8; 4]) -> [u8; 20] { - let mut key = [0u8; 20]; - key[0..16].copy_from_slice(&from_node_id.to_be_bytes()); - key[16..20].copy_from_slice(label); - key - } - - #[inline(always)] - pub fn in_edge_key(to_node_id: &u128, label: &[u8; 4]) -> [u8; 20] { - let mut key = [0u8; 20]; - key[0..16].copy_from_slice(&to_node_id.to_be_bytes()); - key[16..20].copy_from_slice(label); - key - } - - #[inline(always)] - pub fn pack_edge_data(node_id: &u128, edge_id: &u128) -> [u8; 32] { - let mut key = [0u8; 32]; - key[0..16].copy_from_slice(&edge_id.to_be_bytes()); - key[16..32].copy_from_slice(&node_id.to_be_bytes()); - key - } - - #[inline(always)] - /// Returns (node_id, edge_id) - pub fn unpack_adj_edge_data(data: &[u8]) -> Result<(u128, u128), GraphError> { - let edge_id = u128::from_be_bytes( - data[0..16] - .try_into() - .map_err(|_| GraphError::SliceLengthError)?, - ); - let node_id = u128::from_be_bytes( - data[16..32] - .try_into() - .map_err(|_| GraphError::SliceLengthError)?, - ); - Ok((node_id, edge_id)) - } - - pub fn get_u128_from_bytes(bytes: &[u8]) -> Result { - let mut arr = [0u8; 16]; - arr.copy_from_slice(bytes); - let res = u128::from_be_bytes(arr); - Ok(res) - } - - pub fn get_vector(&self, txn: &RoTxn, id: &u128) -> Result { - let vector = self.vectors.get_vector(txn, *id, 0, true)?; - Ok(vector) - } - - fn get_document_text(&self, txn: &RoTxn, doc_id: u128) -> Result { - let node = self.get_node(txn, &doc_id)?; - let mut text = node.label.clone(); - - // Include properties in the text for indexing - if let Some(properties) = node.properties { - for (key, value) in properties { - text.push(' '); - text.push_str(&key); - text.push(' '); - text.push_str(&value.to_string()); - } - } - - Ok(text) - } -} - -impl DBMethods for HelixGraphStorage { - fn create_secondary_index(&mut self, name: &str) -> Result<(), GraphError> { - let mut wtxn = self.graph_env.write_txn()?; - let db = self.graph_env.create_database(&mut wtxn, Some(name))?; - wtxn.commit()?; - self.secondary_indices.insert(name.to_string(), db); - Ok(()) - } - - fn drop_secondary_index(&mut self, name: &str) -> Result<(), GraphError> { - let mut wtxn = self.graph_env.write_txn()?; - let db = self - .secondary_indices - .get(name) - .ok_or(GraphError::New(format!( - "Secondary Index {} not found", - name - )))?; - db.clear(&mut wtxn)?; - wtxn.commit()?; - self.secondary_indices.remove(name); - Ok(()) - } -} - -impl BasicStorageMethods for HelixGraphStorage { - #[inline(always)] - fn get_temp_node<'a>(&self, txn: &'a RoTxn, id: &u128) -> Result<&'a [u8], GraphError> { - match self.nodes_db.get(&txn, Self::node_key(id))? { - Some(data) => Ok(data), - None => Err(GraphError::NodeNotFound), - } - } - - #[inline(always)] - fn get_temp_edge<'a>(&self, txn: &'a RoTxn, id: &u128) -> Result<&'a [u8], GraphError> { - match self.edges_db.get(&txn, Self::edge_key(id))? { - Some(data) => Ok(data), - None => Err(GraphError::EdgeNotFound), - } - } -} - -impl StorageMethods for HelixGraphStorage { - #[inline(always)] - fn check_exists(&self, txn: &RoTxn, id: &u128) -> Result { - // let txn = txn.read_txn(); - let exists = self.nodes_db.get(txn, Self::node_key(id))?.is_some(); - Ok(exists) - } - - #[inline(always)] - fn get_node(&self, txn: &RoTxn, id: &u128) -> Result { - let node = match self.nodes_db.get(txn, Self::node_key(id))? { - Some(data) => data, - None => return Err(GraphError::NodeNotFound), - }; - let node: Node = match Node::decode_node(&node, *id) { - Ok(node) => node, - Err(e) => return Err(e), - }; - Ok(node) - } - - #[inline(always)] - fn get_edge(&self, txn: &RoTxn, id: &u128) -> Result { - let edge = match self.edges_db.get(txn, Self::edge_key(id))? { - Some(data) => data, - None => return Err(GraphError::EdgeNotFound), - }; - let edge: Edge = match Edge::decode_edge(&edge, *id) { - Ok(edge) => edge, - Err(e) => return Err(e), - }; - Ok(edge) - } - - // LEAVE FOR NOW - // fn get_node_by_secondary_index( - // &self, - // txn: &RoTxn, - // index: &str, - // key: &Value, - // ) -> Result { - // let db = self - // .secondary_indices - // .get(index) - // .ok_or(GraphError::New(format!( - // "Secondary Index {} not found", - // index - // )))?; - // let node_id = db - // .get(txn, &bincode::serialize(key)?)? - // .ok_or(GraphError::NodeNotFound)?; - // let node_id = Self::get_u128_from_bytes(&node_id)?; - // self.get_node(txn, &node_id) - // } - - fn drop_node(&self, txn: &mut RwTxn, id: &u128) -> Result<(), GraphError> { - // Get node to get its label - //let node = self.get_node(txn, id)?; - - // Delete outgoing edges - let out_edges = { - let iter = self.out_edges_db.prefix_iter(&txn, &id.to_be_bytes())?; - let capacity = match iter.size_hint() { - (_, Some(upper)) => upper, - (lower, None) => lower, - }; - let mut out_edges = Vec::with_capacity(capacity); - - for result in iter { - let (key, value) = result?; - assert_eq!(key.len(), 20); - let mut label = [0u8; 4]; - label.copy_from_slice(&key[16..20]); - let (_, edge_id) = Self::unpack_adj_edge_data(&value)?; - out_edges.push((edge_id, label)); - } - out_edges - }; - - // Delete incoming edges - - let in_edges = { - let iter = self.in_edges_db.prefix_iter(&txn, &id.to_be_bytes())?; - let capacity = match iter.size_hint() { - (_, Some(c)) => c, - (c, None) => c, - }; - let mut in_edges = Vec::with_capacity(capacity); - - for result in iter { - let (key, value) = result?; - assert_eq!(key.len(), 20); - let mut label = [0u8; 4]; - label.copy_from_slice(&key[16..20]); - let (node_id, edge_id) = Self::unpack_adj_edge_data(&value)?; - in_edges.push((edge_id, label, node_id)); - } - - in_edges - }; - - // Delete all related data - for (out_edge_id, label_bytes) in out_edges.iter() { - // Delete edge data - self.edges_db.delete(txn, &Self::edge_key(out_edge_id))?; - self.out_edges_db - .delete(txn, &Self::out_edge_key(id, label_bytes))?; - } - for (in_edge_id, label_bytes, other_id) in in_edges.iter() { - self.edges_db.delete(txn, &Self::edge_key(in_edge_id))?; - self.in_edges_db - .delete(txn, &Self::in_edge_key(other_id, label_bytes))?; - } - - // Delete node data and label - self.nodes_db.delete(txn, Self::node_key(id))?; - - Ok(()) - } - - fn drop_edge(&self, txn: &mut RwTxn, edge_id: &u128) -> Result<(), GraphError> { - // Get edge data first - let edge_data = match self.edges_db.get(&txn, &Self::edge_key(edge_id))? { - Some(data) => data, - None => return Err(GraphError::EdgeNotFound), - }; - let edge: Edge = bincode::deserialize(edge_data)?; - let label_hash = hash_label(&edge.label, None); - // Delete all edge-related data - self.edges_db.delete(txn, &Self::edge_key(edge_id))?; - self.out_edges_db - .delete(txn, &Self::out_edge_key(&edge.from_node, &label_hash))?; - self.in_edges_db - .delete(txn, &Self::in_edge_key(&edge.to_node, &label_hash))?; - - Ok(()) - } -} diff --git a/helixdb/src/helix_engine/storage_core/storage_methods.rs b/helixdb/src/helix_engine/storage_core/storage_methods.rs deleted file mode 100644 index ddf205f50..000000000 --- a/helixdb/src/helix_engine/storage_core/storage_methods.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::helix_engine::types::GraphError; -use crate::protocol::{ - items::{Edge, Node}, - value::Value, -}; -use heed3::{RoTxn, RwTxn}; - -pub trait DBMethods { - /// Creates a new database with a given name for a secondary index - fn create_secondary_index(&mut self, name: &str) -> Result<(), GraphError>; - - /// Opens a database with a given name for a secondary index - fn drop_secondary_index(&mut self, name: &str) -> Result<(), GraphError>; -} - -pub trait BasicStorageMethods { - /// Gets a node object for a given node id without copying its underlying data. - /// - /// This should only used when fetched data is only needed temporarily - /// as underlying data is pinned. - fn get_temp_node<'a>(&self, txn: &'a RoTxn, id: &u128) -> Result<&'a [u8], GraphError>; - - /// Gets a edge object for a given edge id without copying its underlying data. - /// - /// This should only used when fetched data is only needed temporarily - /// as underlying data is pinned. - fn get_temp_edge<'a>(&self, txn: &'a RoTxn, id: &u128) -> Result<&'a [u8], GraphError>; -} -pub trait StorageMethods { - /// Checks whether an entry with a given id exists. - /// Works for nodes or edges. - fn check_exists(&self, txn: &RoTxn, id: &u128) -> Result; - - /// Gets a node object for a given node id - fn get_node(&self, txn: &RoTxn, id: &u128) -> Result; - /// Gets a edge object for a given edge id - fn get_edge(&self, txn: &RoTxn, id: &u128) -> Result; - - fn drop_node(&self, txn: &mut RwTxn, id: &u128) -> Result<(), GraphError>; - fn drop_edge(&self, txn: &mut RwTxn, id: &u128) -> Result<(), GraphError>; -} - -pub trait SearchMethods { - /// Find shortest path between two nodes - fn shortest_path( - &self, - txn: &RoTxn<'_>, - edge_label: &str, - from_id: &u128, - to_id: &u128, - ) -> Result<(Vec, Vec), GraphError>; - - fn shortest_mutual_path( - &self, - txn: &RoTxn<'_>, - edge_label: &str, - from_id: &u128, - to_id: &u128, - ) -> Result<(Vec, Vec), GraphError>; -} diff --git a/helixdb/src/helix_engine/types.rs b/helixdb/src/helix_engine/types.rs index 26fd59eba..3525ff99e 100644 --- a/helixdb/src/helix_engine/types.rs +++ b/helixdb/src/helix_engine/types.rs @@ -10,28 +10,51 @@ use std::{ str::Utf8Error, string::FromUtf8Error }; +use thiserror::Error; -#[derive(Debug)] +#[derive(Error, Debug)] pub enum GraphError { - Io(std::io::Error), + #[error("IO error: {0}")] + Io(#[from] std::io::Error), + #[error("Graph connection error: {0} {1}")] GraphConnectionError(String, std::io::Error), + #[error("Storage connection error: {0} {1}")] StorageConnectionError(String, std::io::Error), + #[error("Storage error: {0}")] StorageError(String), + #[error("Traversal error: {0}")] TraversalError(String), + #[error("Conversion error: {0}")] ConversionError(String), + #[error("Decode error: {0}")] DecodeError(String), - EdgeNotFound, - NodeNotFound, + #[error("Edge not found")] + EdgeNotFound(u128), + #[error("Node not found")] + NodeNotFound(u128), + #[error("Label not found")] LabelNotFound, + #[error("Wrong traversal value")] + WrongTraversalValue, + #[error("Vector error: {0}")] VectorError(String), + #[error("Default graph error")] Default, + #[error("Graph error: {0}")] New(String), + #[error("No error")] Empty, + #[error("Multiple nodes with same id")] MultipleNodesWithSameId, + #[error("Multiple edges with same id")] MultipleEdgesWithSameId, + #[error("Invalid node")] InvalidNode, + #[error("Config file not found")] ConfigFileNotFound, + #[error("Slice length error")] SliceLengthError, + #[error("Shortest path not found")] ShortestPathNotFound } @@ -47,11 +70,12 @@ impl fmt::Display for GraphError { }, GraphError::TraversalError(msg) => write!(f, "Traversal error: {}", msg), GraphError::StorageError(msg) => write!(f, "Storage error: {}", msg), - GraphError::ConversionError(msg ) => write!(f, "Conversion error: {}", msg), + GraphError::ConversionError(msg) => write!(f, "Conversion error: {}", msg), GraphError::DecodeError(msg) => write!(f, "Decode error: {}", msg), GraphError::EdgeNotFound => write!(f, "Edge not found"), GraphError::NodeNotFound => write!(f, "Node not found"), GraphError::LabelNotFound => write!(f, "Label not found"), + GraphError::WrongTraversalValue => write!(f, "Wrong traversal value"), GraphError::New(msg) => write!(f, "Graph error: {}", msg), GraphError::Default => write!(f, "Graph error"), GraphError::Empty => write!(f, "No Error"), @@ -154,13 +178,19 @@ impl From for GraphError { } } -#[derive(Debug)] +#[derive(Error, Debug)] pub enum VectorError { + #[error("Vector not found: {0}")] VectorNotFound(String), + #[error("Invalid vector length")] InvalidVectorLength, + #[error("Invalid vector data")] InvalidVectorData, + #[error("Entry point not found")] EntryPointNotFound, + #[error("Conversion error: {0}")] ConversionError(String), + #[error("Vector core error: {0}")] VectorCoreError(String), } diff --git a/helixdb/src/helix_engine/vector_core/hnsw.rs b/helixdb/src/helix_engine/vector_core/hnsw.rs index 27e4d0eba..c1ff5220e 100644 --- a/helixdb/src/helix_engine/vector_core/hnsw.rs +++ b/helixdb/src/helix_engine/vector_core/hnsw.rs @@ -38,14 +38,12 @@ pub trait HNSW /// # Returns /// /// An HVector of the data inserted - fn insert( + fn insert( &self, txn: &mut RwTxn, data: &[f64], fields: Option>, - ) -> Result - where - F: Fn(&HVector, &RoTxn) -> bool; + ) -> Result; /// Get all vectors from the index at a specific level /// diff --git a/helixdb/src/helix_engine/vector_core/vector_core.rs b/helixdb/src/helix_engine/vector_core/vector_core.rs index dabadcbc8..867d6c312 100644 --- a/helixdb/src/helix_engine/vector_core/vector_core.rs +++ b/helixdb/src/helix_engine/vector_core/vector_core.rs @@ -215,13 +215,13 @@ impl VectorCore { // TODO: look at using the XOR shift algorithm for random number generation // Storing global rng will not be threadsafe or possible as thread rng needs to be mutable // Should instead using an atomic mutable seed and the XOR shift algorithm - let mut rng = rand::rng(); - let r: f64 = rng.random::(); + let mut rng = rand::thread_rng(); + let r: f64 = rng.gen::(); let level = (-r.ln() * self.config.m_l).floor() as usize; level } - #[inline] + #[inline(always)] fn get_entry_point(&self, txn: &RoTxn) -> Result { let ep_id = self.vectors_db.get(txn, ENTRY_POINT_KEY.as_bytes())?; if let Some(ep_id) = ep_id { @@ -238,7 +238,7 @@ impl VectorCore { } } - #[inline] + #[inline(always)] fn set_entry_point(&self, txn: &mut RwTxn, entry: &HVector) -> Result<(), VectorError> { let entry_key = ENTRY_POINT_KEY.as_bytes().to_vec(); self.vectors_db @@ -487,7 +487,7 @@ impl HNSW for VectorCore { where F: Fn(&HVector, &RoTxn) -> bool, { - let query = HVector::from_slice(0, query.to_vec()); + let mut query = HVector::from_slice(0, query.to_vec()); let mut entry_point = self.get_entry_point(txn)?; @@ -497,7 +497,7 @@ impl HNSW for VectorCore { for level in (1..=curr_level).rev() { let mut nearest = self.search_level( txn, - &query, + &mut query, &mut entry_point, 1, level, @@ -513,7 +513,7 @@ impl HNSW for VectorCore { let mut candidates = self.search_level( txn, - &query, + &mut query, &mut entry_point, ef, 0, @@ -538,17 +538,13 @@ impl HNSW for VectorCore { Ok(results) } - fn insert( + fn insert( &self, txn: &mut RwTxn, data: &[f64], fields: Option>, - ) -> Result - where - F: Fn(&HVector, &RoTxn) -> bool, - { + ) -> Result { let new_level = self.get_new_level(); - let mut query = HVector::from_slice(0, data.to_vec()); self.put_vector(txn, &query)?; @@ -569,39 +565,39 @@ impl HNSW for VectorCore { let l = entry_point.get_level(); let mut curr_ep = entry_point; for level in (new_level + 1..=l).rev() { - let nearest = self.search_level::(txn, &query, &mut curr_ep, 1, level, None)?; - curr_ep = nearest.peek().unwrap().clone(); + let mut nearest = self.search_level(txn, &mut query, &mut curr_ep, 1, level, Option::<&[fn(&HVector, &RoTxn) -> bool]>::None)?; + curr_ep = nearest.pop().unwrap().clone(); } for level in (0..=l.min(new_level)).rev() { - let nearest = self.search_level::( + let mut nearest = self.search_level( txn, - &query, + &mut query, &mut curr_ep, self.config.ef_construct, level, - None, + Option::<&[fn(&HVector, &RoTxn) -> bool]>::None, )?; - curr_ep = nearest.peek().unwrap().clone(); + curr_ep = nearest.pop().unwrap().clone(); - let neighbors = self.select_neighbors::(txn, &query, nearest, level, true, None)?; + let neighbors = self.select_neighbors(txn, &query, nearest, level, true, Option::<&[fn(&HVector, &RoTxn) -> bool]>::None)?; self.set_neighbours(txn, query.get_id(), &neighbors, level)?; for e in neighbors { let id = e.get_id(); - let e_conns = self.get_neighbors::(txn, id, level, None)?; + let e_conns = self.get_neighbors(txn, id, level, Option::<&[fn(&HVector, &RoTxn) -> bool]>::None)?; if e_conns.len() > if level == 0 { self.config.m_max_0 } else { - self.config.m_max_0 + self.config.m } { let e_conns = BinaryHeap::from(e_conns); let e_new_conn = - self.select_neighbors::(txn, &query, e_conns, level, true, None)?; + self.select_neighbors(txn, &query, e_conns, level, true, Option::<&[fn(&HVector, &RoTxn) -> bool]>::None)?; self.set_neighbours(txn, id, &e_new_conn, level)?; } } diff --git a/helixdb/src/helix_gateway/connection/connection.rs b/helixdb/src/helix_gateway/connection/connection.rs index 856139bf2..5a8b80993 100644 --- a/helixdb/src/helix_gateway/connection/connection.rs +++ b/helixdb/src/helix_gateway/connection/connection.rs @@ -2,6 +2,7 @@ use crate::helix_engine::graph_core::graph_core::HelixGraphEngine; use crate::helix_engine::types::GraphError; use crate::helix_gateway::{router::router::HelixRouter, thread_pool::thread_pool::ThreadPool}; use crate::helix_runtime::AsyncRuntime; +use crate::helix_storage::lmdb_storage::LmdbStorage; use crate::helix_transport::{Listener, Transport}; use chrono::{DateTime, Utc}; use std::{ @@ -38,7 +39,7 @@ where { pub fn new( address: &str, - graph: Arc, + graph: Arc>, size: usize, router: HelixRouter, runtime: R, diff --git a/helixdb/src/helix_gateway/gateway.rs b/helixdb/src/helix_gateway/gateway.rs index 3199c406f..3f599f9a0 100644 --- a/helixdb/src/helix_gateway/gateway.rs +++ b/helixdb/src/helix_gateway/gateway.rs @@ -5,6 +5,7 @@ use crate::helix_runtime::AsyncRuntime; use super::router::router::{HandlerFn, HelixRouter}; use crate::{ helix_engine::graph_core::graph_core::HelixGraphEngine, helix_gateway::mcp::mcp::MCPHandlerFn, + helix_storage::lmdb_storage::LmdbStorage, }; use crate::helix_transport::Transport; @@ -31,7 +32,7 @@ where { pub async fn new( address: &str, - graph: Arc, + graph: Arc>, size: usize, routes: Option>, mcp_routes: Option>, diff --git a/helixdb/src/helix_gateway/mcp/mcp.rs b/helixdb/src/helix_gateway/mcp/mcp.rs index fa9b8bec2..96e05ca2a 100644 --- a/helixdb/src/helix_gateway/mcp/mcp.rs +++ b/helixdb/src/helix_gateway/mcp/mcp.rs @@ -12,6 +12,8 @@ use std::{ use get_routes::{local_handler, mcp_handler}; use heed3::{AnyTls, RoTxn}; use serde::Deserialize; +use tokio::sync::mpsc; +use tokio::sync::mpsc::{Receiver, Sender}; use crate::{ helix_engine::{ @@ -25,13 +27,13 @@ use crate::{ }, traversal_iter::RoTraversalIterator, }, - storage_core::storage_core::HelixGraphStorage, types::GraphError, }, helix_gateway::{ mcp::tools::{ToolArgs, ToolCalls}, router::router::HandlerInput, }, + helix_storage::lmdb_storage::LmdbStorage, protocol::{ items::v6_uuid, label_hash::hash_label, request::Request, response::Response, return_values::ReturnValue, @@ -71,7 +73,7 @@ impl McpConnections { } } pub struct McpBackend { - pub db: Arc, + pub db: Arc, } #[derive(Debug, Deserialize)] @@ -82,7 +84,7 @@ pub struct ToolCallRequest { } impl McpBackend { - pub fn new(db: Arc) -> Self { + pub fn new(db: Arc) -> Self { Self { db } } } @@ -91,24 +93,26 @@ pub struct MCPConnection { pub connection_id: String, pub connection_addr: String, pub connection_port: u16, + pub sender: Sender, + pub receiver: Receiver, pub iter: IntoIter, } -// pub struct McpIter { -// pub iter: I, -// } - impl MCPConnection { - pub fn new( + pub async fn new( connection_id: String, connection_addr: String, connection_port: u16, iter: IntoIter, ) -> Self { + let (tx, rx) = mpsc::channel::(100); + Self { connection_id, connection_addr, connection_port, + sender: tx, + receiver: rx, iter, } } @@ -163,7 +167,7 @@ pub fn call_tool<'a>( None => return Err(GraphError::Default), }; drop(connections); - let txn = input.mcp_backend.db.graph_env.read_txn()?; + let txn = input.mcp_backend.db.ro_txn()?; let result = input.mcp_backend.call(&txn, &connection, data.tool)?; @@ -194,12 +198,20 @@ pub fn init<'a>(input: &'a mut MCPToolInput, response: &mut Response) -> Result< let connection_id = uuid::Uuid::from_u128(v6_uuid()).to_string(); let mut connections = input.mcp_connections.lock().unwrap(); - connections.add_connection(MCPConnection::new( - connection_id.clone(), - data.connection_addr, - data.connection_port, - vec![].into_iter(), - )); + + // This is now an async function, but we are in a sync context. + // For the temporary fix, I will block on the future. + // This will be properly handled in the full async refactor of McpBackend. + let connection = tokio::runtime::Runtime::new().unwrap().block_on( + MCPConnection::new( + connection_id.clone(), + data.connection_addr, + data.connection_port, + vec![].into_iter(), + ) + ); + connections.add_connection(connection); + drop(connections); response.body = sonic_rs::to_vec(&ReturnValue::from(connection_id)).unwrap(); diff --git a/helixdb/src/helix_gateway/mcp/tools.rs b/helixdb/src/helix_gateway/mcp/tools.rs index 52a8eee77..db81b6c51 100644 --- a/helixdb/src/helix_gateway/mcp/tools.rs +++ b/helixdb/src/helix_gateway/mcp/tools.rs @@ -7,14 +7,13 @@ use crate::helix_engine::graph_core::ops::source::add_e::EdgeType; use crate::helix_engine::graph_core::ops::source::e_from_type::{EFromType, EFromTypeAdapter}; use crate::helix_engine::graph_core::ops::source::n_from_type::{NFromType, NFromTypeAdapter}; use crate::helix_engine::graph_core::ops::tr_val::{Traversable, TraversalVal}; -use crate::helix_engine::storage_core::storage_core::HelixGraphStorage; use crate::helix_engine::types::GraphError; use crate::helix_gateway::mcp::mcp::{MCPConnection, McpBackend}; use crate::helix_gateway::router::router::HandlerInput; +use crate::helix_storage::lmdb_storage::{LmdbRoTxn, LmdbStorage}; use crate::protocol::label_hash::hash_label; use crate::protocol::response::Response; use get_routes::local_handler; -use heed3::RoTxn; use serde::{Deserialize, Deserializer}; use std::collections::HashMap; use std::sync::Arc; @@ -48,7 +47,7 @@ pub enum ToolArgs { pub(crate) trait ToolCalls<'a> { fn call( &'a self, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, connection_id: &'a MCPConnection, args: ToolArgs, ) -> Result, GraphError>; @@ -57,7 +56,7 @@ pub(crate) trait ToolCalls<'a> { impl<'a> ToolCalls<'a> for McpBackend { fn call( &'a self, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, connection: &'a MCPConnection, args: ToolArgs, ) -> Result, GraphError> { @@ -87,14 +86,14 @@ trait McpTools<'a> { connection: &'a MCPConnection, edge_label: &'a str, edge_type: &'a EdgeType, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; fn out_e_step( &'a self, connection: &'a MCPConnection, edge_label: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; fn in_step( @@ -102,32 +101,32 @@ trait McpTools<'a> { connection: &'a MCPConnection, edge_label: &'a str, edge_type: &'a EdgeType, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; fn in_e_step( &'a self, connection: &'a MCPConnection, edge_label: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; fn n_from_type( &'a self, node_type: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; fn e_from_type( &'a self, edge_type: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError>; /// filters items based on properies and traversal existence fn filter_items( &'a self, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, connection: &'a MCPConnection, properties: Option>, filter_traversals: Option>, @@ -140,7 +139,7 @@ impl<'a> McpTools<'a> for McpBackend { connection: &'a MCPConnection, edge_label: &'a str, edge_type: &'a EdgeType, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -149,7 +148,7 @@ impl<'a> McpTools<'a> for McpBackend { .clone() .filter_map(move |item| { let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::out_edge_key(&item.id(), &edge_label_hash); + let prefix = LmdbStorage::out_edge_key(&item.id(), &edge_label_hash); match db .out_edges_db .lazily_decode_data() @@ -185,7 +184,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, connection: &'a MCPConnection, edge_label: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -194,7 +193,7 @@ impl<'a> McpTools<'a> for McpBackend { .clone() .filter_map(move |item| { let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::out_edge_key(&item.id(), &edge_label_hash); + let prefix = LmdbStorage::out_edge_key(&item.id(), &edge_label_hash); match db .out_edges_db .lazily_decode_data() @@ -225,7 +224,7 @@ impl<'a> McpTools<'a> for McpBackend { connection: &'a MCPConnection, edge_label: &'a str, edge_type: &'a EdgeType, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -234,7 +233,7 @@ impl<'a> McpTools<'a> for McpBackend { .clone() .filter_map(move |item| { let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::in_edge_key(&item.id(), &edge_label_hash); + let prefix = LmdbStorage::in_edge_key(&item.id(), &edge_label_hash); match db .in_edges_db .lazily_decode_data() @@ -270,7 +269,7 @@ impl<'a> McpTools<'a> for McpBackend { &'a self, connection: &'a MCPConnection, edge_label: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); @@ -279,7 +278,7 @@ impl<'a> McpTools<'a> for McpBackend { .clone() .filter_map(move |item| { let edge_label_hash = hash_label(edge_label, None); - let prefix = HelixGraphStorage::in_edge_key(&item.id(), &edge_label_hash); + let prefix = LmdbStorage::in_edge_key(&item.id(), &edge_label_hash); match db .in_edges_db .lazily_decode_data() @@ -308,12 +307,12 @@ impl<'a> McpTools<'a> for McpBackend { fn n_from_type( &'a self, node_type: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); let iter = NFromType { - iter: db.nodes_db.lazily_decode_data().iter(txn).unwrap(), + iter: db.nodes_db.iter(txn).unwrap(), label: node_type, }; @@ -325,12 +324,12 @@ impl<'a> McpTools<'a> for McpBackend { fn e_from_type( &'a self, edge_type: &'a str, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, ) -> Result, GraphError> { let db = Arc::clone(&self.db); let iter = EFromType { - iter: db.edges_db.lazily_decode_data().iter(txn).unwrap(), + iter: db.edges_db.iter(txn).unwrap(), label: edge_type, }; @@ -341,7 +340,7 @@ impl<'a> McpTools<'a> for McpBackend { fn filter_items( &'a self, - txn: &'a RoTxn, + txn: &'a LmdbRoTxn, connection: &'a MCPConnection, properties: Option>, filter_traversals: Option>, diff --git a/helixdb/src/helix_gateway/router/router.rs b/helixdb/src/helix_gateway/router/router.rs index 75bd34909..32c15d2ea 100644 --- a/helixdb/src/helix_gateway/router/router.rs +++ b/helixdb/src/helix_gateway/router/router.rs @@ -10,6 +10,7 @@ use crate::{ helix_engine::{graph_core::graph_core::HelixGraphEngine, types::GraphError}, helix_gateway::mcp::mcp::{MCPHandlerFn, MCPToolInput, McpConnections}, + helix_storage::lmdb_storage::LmdbStorage, }; use core::fmt; use std::{collections::HashMap, sync::Arc}; @@ -18,7 +19,7 @@ use crate::protocol::{request::Request, response::Response}; pub struct HandlerInput { pub request: Request, - pub graph: Arc, + pub graph: Arc>, } // basic type for function pointer @@ -57,14 +58,8 @@ impl HelixRouter { routes: Option>, mcp_routes: Option>, ) -> Self { - let rts = match routes { - Some(routes) => routes, - None => HashMap::new(), - }; - let mcp_rts = match mcp_routes { - Some(routes) => routes, - None => HashMap::new(), - }; + let rts = routes.unwrap_or_default(); + let mcp_rts = mcp_routes.unwrap_or_default(); Self { routes: rts, mcp_routes: mcp_rts, @@ -91,7 +86,7 @@ impl HelixRouter { /// * `Err(RouterError)` if there was an error handling the request pub fn handle( &self, - graph_access: Arc, + graph_access: Arc>, request: Request, response: &mut Response, ) -> Result<(), GraphError> { @@ -108,15 +103,15 @@ impl HelixRouter { if let Some(mcp_handler) = self.mcp_routes.get(&route_key) { let mut mcp_input = MCPToolInput { request, - mcp_backend: Arc::clone(&graph_access.mcp_backend.as_ref().unwrap()), - mcp_connections: Arc::clone(&graph_access.mcp_connections.as_ref().unwrap()), + mcp_backend: Arc::clone(graph_access.mcp_backend.as_ref().unwrap()), + mcp_connections: Arc::clone(graph_access.mcp_connections.as_ref().unwrap()), }; return mcp_handler(&mut mcp_input, response); }; response.status = 404; response.body = b"404 - Not Found".to_vec(); - return Ok(()); + Ok(()) } } diff --git a/helixdb/src/helix_gateway/thread_pool/thread_pool.rs b/helixdb/src/helix_gateway/thread_pool/thread_pool.rs index bb80a915d..e80ee5f9a 100644 --- a/helixdb/src/helix_gateway/thread_pool/thread_pool.rs +++ b/helixdb/src/helix_gateway/thread_pool/thread_pool.rs @@ -7,6 +7,7 @@ use crate::helix_gateway::router::router::{HelixRouter, RouterError}; use crate::protocol::request::Request; use crate::protocol::response::Response; +use crate::helix_storage::lmdb_storage::LmdbStorage; use crate::helix_transport::Stream; pub struct Worker { @@ -19,7 +20,7 @@ pub struct Worker { impl Worker { fn new( id: usize, - graph_access: Arc, + graph_access: Arc>, router: Arc, rx: Receiver, runtime: R, @@ -85,7 +86,7 @@ pub struct ThreadPool { impl ThreadPool { pub fn new( size: usize, - graph: Arc, + graph: Arc>, router: Arc, runtime: R, ) -> Result, RouterError> { diff --git a/helixdb/src/helix_storage/lmdb_storage.rs b/helixdb/src/helix_storage/lmdb_storage.rs new file mode 100644 index 000000000..9ce395057 --- /dev/null +++ b/helixdb/src/helix_storage/lmdb_storage.rs @@ -0,0 +1,682 @@ +use std::collections::HashMap; +use std::fs; +use std::ops::{Deref, DerefMut}; +use std::path::Path; + +use heed3::byteorder::BE; +use heed3::{ + types::*, Database, DatabaseFlags, Env, EnvOpenOptions, RoTxn as HeedRoTxn, + RwTxn as HeedRwTxn, WithTls, +}; + +use super::{DbRoTxn, DbRwTxn, Storage}; +use crate::helix_engine::bm25::bm25::{BM25, BM25Flatten, HBM25Config}; +use crate::helix_engine::graph_core::config::Config; +use crate::helix_engine::types::GraphError; +use crate::helix_engine::vector_core::hnsw::HNSW; +use crate::helix_engine::vector_core::vector::HVector; +use crate::helix_engine::vector_core::vector_core::{HNSWConfig, VectorCore}; +use crate::protocol::items::{Edge, Node, v6_uuid}; +use crate::protocol::label_hash::hash_label; +use crate::protocol::value::Value; +use serde::Serialize; +use std::borrow::Cow; +use heed3::{BytesDecode, BytesEncode}; + +// region: Codecs +struct NodeCodec; + +impl<'a> BytesEncode<'a> for NodeCodec { + type EItem = Node; + + fn bytes_encode(item: &'a Self::EItem) -> Result, Box> { + item.encode_node().map(Cow::Owned).map_err(|e| e.into()) + } +} + +impl<'a> BytesDecode<'a> for NodeCodec { + type DItem = Node; + + fn bytes_decode(bytes: &'a [u8]) -> Result> { + // The ID is not stored in the value, it's the key. We pass a dummy 0. + Node::decode_node(bytes, 0).map_err(|e| e.into()) + } +} + +struct EdgeCodec; + +impl<'a> BytesEncode<'a> for EdgeCodec { + type EItem = Edge; + + fn bytes_encode(item: &'a Self::EItem) -> Result, Box> { + item.encode_edge().map(Cow::Owned).map_err(|e| e.into()) + } +} + +impl<'a> BytesDecode<'a> for EdgeCodec { + type DItem = Edge; + + fn bytes_decode(bytes: &'a [u8]) -> Result> { + // The ID is not stored in the value, it's the key. We pass a dummy 0. + Edge::decode_edge(bytes, 0).map_err(|e| e.into()) + } +} + +// endregion + +// region: Transaction Wrappers +pub struct LmdbRoTxn<'a>(pub HeedRoTxn<'a>); + +impl<'a> DbRoTxn<'a> for LmdbRoTxn<'a> {} + +impl<'a> Deref for LmdbRoTxn<'a> { + type Target = HeedRoTxn<'a>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +pub struct LmdbRwTxn<'a>(pub HeedRwTxn<'a>); + +impl<'a> DbRoTxn<'a> for LmdbRwTxn<'a> {} + +impl<'a> DbRwTxn<'a> for LmdbRwTxn<'a> { + fn commit(self) -> Result<(), GraphError> { + self.0.commit().map_err(GraphError::from) + } +} + +impl<'a> Deref for LmdbRwTxn<'a> { + type Target = HeedRwTxn<'a>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl<'a> DerefMut for LmdbRwTxn<'a> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl<'a, 'b> From<&'a mut LmdbRwTxn<'b>> for LmdbRoTxn<'a> { + fn from(txn: &'a mut LmdbRwTxn<'b>) -> Self { + LmdbRoTxn(txn.0.deref()) + } +} +// endregion + +// region: LmdbStorage Definition +pub struct LmdbStorage { + pub graph_env: Env, + pub nodes_db: Database, NodeCodec>, + pub edges_db: Database, EdgeCodec>, + pub out_edges_db: Database, + pub in_edges_db: Database, + pub secondary_indices: HashMap>>, + pub vectors: VectorCore, + pub bm25: HBM25Config, +} + +impl LmdbStorage { + pub fn new(path: &str, config: Config) -> Result { + fs::create_dir_all(path)?; + + let db_size = if config.db_max_size_gb.unwrap_or(100) >= 9999 { + 9998 + } else { + config.db_max_size_gb.unwrap_or(100) + }; + + let graph_env = unsafe { + EnvOpenOptions::new() + .map_size(db_size * 1024 * 1024 * 1024) + .max_dbs(20) + .max_readers(200) + .open(Path::new(path))? + }; + + let mut wtxn = graph_env.write_txn()?; + + let nodes_db = graph_env + .database_options() + .types::, NodeCodec>() + .name("nodes") + .create(&mut wtxn)?; + let edges_db = graph_env + .database_options() + .types::, EdgeCodec>() + .name("edges") + .create(&mut wtxn)?; + let out_edges_db: Database = graph_env + .database_options() + .types::() + .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) + .name("out_edges") + .create(&mut wtxn)?; + let in_edges_db: Database = graph_env + .database_options() + .types::() + .flags(DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED) + .name("in_edges") + .create(&mut wtxn)?; + + let mut secondary_indices = HashMap::new(); + if let Some(indexes) = config.graph_config.secondary_indices { + for index in indexes { + secondary_indices.insert( + index.clone(), + graph_env + .database_options() + .types::>() + .flags(DatabaseFlags::DUP_SORT) + .name(&index) + .create(&mut wtxn)?, + ); + } + } + + let vectors = VectorCore::new( + &graph_env, + &mut wtxn, + HNSWConfig::new( + config.vector_config.m, + config.vector_config.ef_construction, + config.vector_config.ef_search, + ), + )?; + let bm25 = HBM25Config::new(&graph_env, &mut wtxn)?; + + wtxn.commit()?; + Ok(Self { + graph_env, + nodes_db, + edges_db, + out_edges_db, + in_edges_db, + secondary_indices, + vectors, + bm25, + }) + } + + // Helper methods from HelixGraphStorage + #[inline(always)] + pub fn node_key(id: &u128) -> &u128 { + id + } + + #[inline(always)] + pub fn edge_key(id: &u128) -> &u128 { + id + } + + #[inline(always)] + pub fn out_edge_key(from_node_id: &u128, label: &[u8; 4]) -> [u8; 20] { + let mut key = [0u8; 20]; + key[0..16].copy_from_slice(&from_node_id.to_be_bytes()); + key[16..20].copy_from_slice(label); + key + } + + #[inline(always)] + pub fn in_edge_key(to_node_id: &u128, label: &[u8; 4]) -> [u8; 20] { + let mut key = [0u8; 20]; + key[0..16].copy_from_slice(&to_node_id.to_be_bytes()); + key[16..20].copy_from_slice(label); + key + } + + #[inline(always)] + pub fn pack_edge_data(node_id: &u128, edge_id: &u128) -> [u8; 32] { + let mut key = [0u8; 32]; + key[0..16].copy_from_slice(&edge_id.to_be_bytes()); + key[16..32].copy_from_slice(&node_id.to_be_bytes()); + key + } + + #[inline(always)] + pub fn unpack_adj_edge_data(data: &[u8]) -> Result<(u128, u128), GraphError> { + let edge_id = u128::from_be_bytes( + data[0..16] + .try_into() + .map_err(|_| GraphError::SliceLengthError)?, + ); + let node_id = u128::from_be_bytes( + data[16..32] + .try_into() + .map_err(|_| GraphError::SliceLengthError)?, + ); + Ok((node_id, edge_id)) + } +} +// endregion + +// region: Storage Trait Implementation +impl Storage for LmdbStorage { + type RoTxn<'a> = LmdbRoTxn<'a>; + type RwTxn<'a> = LmdbRwTxn<'a>; + + fn ro_txn(&self) -> Result, GraphError> { + self.graph_env.read_txn().map(LmdbRoTxn).map_err(Into::into) + } + + fn rw_txn(&self) -> Result, GraphError> { + self.graph_env.write_txn().map(LmdbRwTxn).map_err(Into::into) + } + + fn create_secondary_index(&mut self, name: &str) -> Result<(), GraphError> { + let mut wtxn = self.graph_env.write_txn()?; + let db = self.graph_env.create_database(&mut wtxn, Some(name))?; + wtxn.commit()?; + self.secondary_indices.insert(name.to_string(), db); + Ok(()) + } + + fn drop_secondary_index(&mut self, name: &str) -> Result<(), GraphError> { + let mut wtxn = self.graph_env.write_txn()?; + let db = self + .secondary_indices + .get(name) + .ok_or_else(|| GraphError::New(format!("Secondary Index {} not found", name)))?; + db.clear(&mut wtxn)?; + wtxn.commit()?; + self.secondary_indices.remove(name); + Ok(()) + } + + fn check_exists<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result { + let exists = self.nodes_db.get(txn, Self::node_key(id))?.is_some(); + Ok(exists) + } + + fn get_node<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result { + self.nodes_db + .get(txn, id)? + .map(|mut node| { + node.id = *id; + node + }) + .ok_or_else(|| GraphError::NodeNotFound(*id)) + } + + fn get_edge<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result { + self.edges_db + .get(txn, id)? + .map(|mut edge| { + edge.id = *id; + edge + }) + .ok_or_else(|| GraphError::EdgeNotFound(*id)) + } + + fn drop_node<'a>(&self, txn: &mut Self::RwTxn<'a>, id: &u128) -> Result<(), GraphError> { + let out_edges = { + let iter = self.out_edges_db.prefix_iter(txn, &id.to_be_bytes())?; + let mut out_edges = Vec::with_capacity(iter.size_hint().0); + for result in iter { + let (key, value) = result?; + let mut label = [0u8; 4]; + label.copy_from_slice(&key[16..20]); + let (_, edge_id) = Self::unpack_adj_edge_data(value)?; + out_edges.push((edge_id, label)); + } + out_edges + }; + + let in_edges = { + let iter = self.in_edges_db.prefix_iter(txn, &id.to_be_bytes())?; + let mut in_edges = Vec::with_capacity(iter.size_hint().0); + for result in iter { + let (key, value) = result?; + let mut label = [0u8; 4]; + label.copy_from_slice(&key[16..20]); + let (node_id, edge_id) = Self::unpack_adj_edge_data(value)?; + in_edges.push((edge_id, label, node_id)); + } + in_edges + }; + + for (out_edge_id, label_bytes) in out_edges.iter() { + self.edges_db.delete(txn, Self::edge_key(out_edge_id))?; + self.out_edges_db + .delete(txn, &Self::out_edge_key(id, label_bytes))?; + } + for (in_edge_id, label_bytes, other_id) in in_edges.iter() { + self.edges_db.delete(txn, Self::edge_key(in_edge_id))?; + self.in_edges_db + .delete(txn, &Self::in_edge_key(other_id, label_bytes))?; + } + + self.nodes_db.delete(txn, Self::node_key(id))?; + Ok(()) + } + + fn drop_edge<'a>(&self, txn: &mut Self::RwTxn<'a>, edge_id: &u128) -> Result<(), GraphError> { + let edge_data = self + .edges_db + .get(txn, Self::edge_key(edge_id))? + .ok_or(GraphError::EdgeNotFound)?; + let edge: Edge = bincode::deserialize(edge_data)?; + let label_hash = hash_label(&edge.label, None); + + self.edges_db.delete(txn, Self::edge_key(edge_id))?; + self.out_edges_db + .delete(txn, &Self::out_edge_key(&edge.from_node, &label_hash))?; + self.in_edges_db + .delete(txn, &Self::in_edge_key(&edge.to_node, &label_hash))?; + + Ok(()) + } + + fn update_node<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + id: &u128, + properties: &Value, + ) -> Result { + let mut old_node = self.get_node(&*txn, id)?; + if let Value::Object(props) = properties { + old_node.properties = Some(props.clone()); + } + self.nodes_db.put(txn, id, &old_node)?; + Ok(old_node) + } + + fn update_edge<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + id: &u128, + properties: &Value, + ) -> Result { + let mut old_edge = self.get_edge(&*txn, id)?; + if let Value::Object(props) = properties { + old_edge.properties = Some(props.clone()); + } + self.edges_db.put(txn, id, &old_edge)?; + Ok(old_edge) + } + + fn add_edge<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + label: &str, + properties: Option>, + from_node: u128, + to_node: u128, + ) -> Result { + // TODO: check if nodes exist first + + let edge = Edge { + id: v6_uuid(), + label: label.to_string(), + properties: properties.map(|props| props.into_iter().collect()), + from_node, + to_node, + }; + + let bytes = edge.encode_edge()?; + self.edges_db + .put(txn, &Self::edge_key(&edge.id), &bytes)?; + + let label_hash = hash_label(edge.label.as_str(), None); + + self.out_edges_db.put( + txn, + &Self::out_edge_key(&from_node, &label_hash), + &Self::pack_edge_data(&to_node, &edge.id), + )?; + + self.in_edges_db.put( + txn, + &Self::in_edge_key(&to_node, &label_hash), + &Self::pack_edge_data(&from_node, &edge.id), + )?; + + Ok(edge) + } + + fn get_out_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + ) -> Result, GraphError> { + let out_prefix = LmdbStorage::out_edge_key(from_id, &hash_label(edge_label, None)); + let mut nodes = Vec::new(); + + for result in self.out_edges_db.prefix_iter(txn, &out_prefix)? { + let (_, value) = result?; + let (to_node_id, _) = LmdbStorage::unpack_adj_edge_data(value)?; + let node = self.get_node(txn, &to_node_id)?; + nodes.push(node); + } + + Ok(nodes) + } + + fn get_in_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + to_id: &u128, + ) -> Result, GraphError> { + let in_prefix = LmdbStorage::in_edge_key(to_id, &hash_label(edge_label, None)); + let mut nodes = Vec::new(); + + for result in self.in_edges_db.prefix_iter(txn, &in_prefix)? { + let (_, value) = result?; + let (from_node_id, _) = LmdbStorage::unpack_adj_edge_data(value)?; + let node = self.get_node(txn, &from_node_id)?; + nodes.push(node); + } + + Ok(nodes) + } + + fn get_out_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + ) -> Result, GraphError> { + let out_prefix = LmdbStorage::out_edge_key(from_id, &hash_label(edge_label, None)); + let mut edges = Vec::new(); + + for result in self.out_edges_db.prefix_iter(txn, &out_prefix)? { + let (_, value) = result?; + let (_, edge_id) = LmdbStorage::unpack_adj_edge_data(value)?; + let edge = self.get_edge(txn, &edge_id)?; + edges.push(edge); + } + + Ok(edges) + } + + fn get_in_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + to_id: &u128, + ) -> Result, GraphError> { + let in_prefix = LmdbStorage::in_edge_key(to_id, &hash_label(edge_label, None)); + let mut edges = Vec::new(); + + for result in self.in_edges_db.prefix_iter(txn, &in_prefix)? { + let (_, value) = result?; + let (_, edge_id) = LmdbStorage::unpack_adj_edge_data(value)?; + let edge = self.get_edge(txn, &edge_id)?; + edges.push(edge); + } + + Ok(edges) + } + + fn shortest_path<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + to_id: &u128, + ) -> Result<(Vec, Vec), GraphError> { + let mut queue = std::collections::VecDeque::with_capacity(32); + let mut visited = std::collections::HashSet::with_capacity(64); + let mut parent: std::collections::HashMap = + std::collections::HashMap::with_capacity(32); + queue.push_back(*from_id); + visited.insert(*from_id); + + while let Some(current_id) = queue.pop_front() { + let out_prefix = + LmdbStorage::out_edge_key(¤t_id, &hash_label(edge_label, None)); + + for result in self.out_edges_db.prefix_iter(txn, &out_prefix)? { + let (_, value) = result?; + let (to_node, edge_id) = LmdbStorage::unpack_adj_edge_data(value)?; + + if !visited.contains(&to_node) { + visited.insert(to_node); + let edge = self.get_edge(txn, &edge_id)?; + parent.insert(to_node, (current_id, edge)); + + if to_node == *to_id { + let mut nodes = Vec::with_capacity(parent.len()); + let mut edges = Vec::with_capacity(parent.len() - 1); + let mut current = to_id; + + while current != from_id { + nodes.push(self.get_node(txn, current)?); + let (prev_node, edge) = &parent[current]; + edges.push(edge.clone()); + current = prev_node; + } + nodes.push(self.get_node(txn, from_id)?); + nodes.reverse(); + edges.reverse(); + return Ok((nodes, edges)); + } + queue.push_back(to_node); + } + } + } + Err(GraphError::ShortestPathNotFound) + } + + fn shortest_mutual_path<'a>( + &self, + _txn: &Self::RoTxn<'a>, + _edge_label: &str, + _from_id: &u128, + _to_id: &u128, + ) -> Result<(Vec, Vec), GraphError> { + unimplemented!("shortest_mutual_path is not part of the storage trait directly yet") + } + + fn node_from_index<'a, K>( + &self, + txn: &Self::RoTxn<'a>, + index: &str, + key: K, + ) -> Result, GraphError> + where + K: Into + Serialize, + { + let db = self.get_node_index_db(index)?; + let key_bytes = bincode::serialize(&key.into())?; + + let result = db.get(txn, &key_bytes)?; + + if let Some(node_id_bytes) = result { + let node_id = u128::from_be_bytes(node_id_bytes.try_into().map_err(|_| { + GraphError::ConversionError("Invalid node ID length in index".to_string()) + })?); + return self.get_node(txn, &node_id).map(Some); + } + + Ok(None) + } + + fn index_node<'a, K>( + &self, + txn: &mut Self::RwTxn<'a>, + index: &str, + key: K, + node: &Node, + ) -> Result<(), GraphError> + where + K: Into + Serialize, + { + let db = self.get_node_index_db(index)?; + let key_bytes = bincode::serialize(&key.into())?; + db.put(txn, &key_bytes, &node.id)?; + Ok(()) + } + + fn add_node<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + label: &str, + properties: Option>, + secondary_indices: Option<&[&str]>, + ) -> Result { + let node = Node { + id: v6_uuid(), + label: label.to_string(), + properties: properties.map(|props| props.into_iter().collect()), + }; + + let bytes = node.encode_node()?; + self.nodes_db.put(txn, &node.id, &bytes)?; + + if let Some(indices) = secondary_indices { + for index_name in indices { + let db = self.secondary_indices.get(*index_name).ok_or_else(|| { + GraphError::New(format!("Secondary Index {} not found", index_name)) + })?; + + if let Some(value) = node.properties.as_ref().and_then(|p| p.get(*index_name)) { + let key_bytes = bincode::serialize(value)?; + db.put(txn, &key_bytes, &node.id.to_be_bytes())?; + } + } + } + + if let Some(props) = &node.properties { + let data = props.flatten_bm25(); + if !data.is_empty() { + self.bm25.insert_doc(txn, node.id, &data)?; + } + } + + Ok(node) + } + + fn get_all_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + ) -> Result> + 'a>, GraphError> { + Ok(Box::new(self.nodes_db.iter(txn)?.map(|item| { + item.map(|(id, mut node)| { + node.id = id; + node + }) + .map_err(GraphError::from) + }))) + } + + fn get_all_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + ) -> Result> + 'a>, GraphError> { + Ok(Box::new(self.edges_db.iter(txn)?.map(|item| { + item.map(|(id, mut edge)| { + edge.id = id; + edge + }) + .map_err(GraphError::from) + }))) + } +} +// endregion \ No newline at end of file diff --git a/helixdb/src/helix_storage/mod.rs b/helixdb/src/helix_storage/mod.rs new file mode 100644 index 000000000..2aa12f745 --- /dev/null +++ b/helixdb/src/helix_storage/mod.rs @@ -0,0 +1,131 @@ +pub mod lmdb_storage; + +use crate::helix_engine::types::GraphError; +use crate::protocol::items::{Edge, Node}; +use crate::protocol::value::Value; +use serde::Serialize; + +pub trait DbRoTxn<'a>: Sized {} + +pub trait DbRwTxn<'a>: DbRoTxn<'a> { + fn commit(self) -> Result<(), GraphError>; +} + +pub trait Storage: Send + Sync + 'static { + type RoTxn<'a>: DbRoTxn<'a> + From<&'a mut Self::RwTxn<'a>>; + type RwTxn<'a>: DbRwTxn<'a>; + + fn ro_txn(&self) -> Result, GraphError>; + fn rw_txn(&self) -> Result, GraphError>; + + fn create_secondary_index(&mut self, name: &str) -> Result<(), GraphError>; + fn drop_secondary_index(&mut self, name: &str) -> Result<(), GraphError>; + + fn check_exists<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result; + fn get_node<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result; + fn get_edge<'a>(&self, txn: &Self::RoTxn<'a>, id: &u128) -> Result; + fn drop_node<'a>(&self, txn: &mut Self::RwTxn<'a>, id: &u128) -> Result<(), GraphError>; + fn drop_edge<'a>(&self, txn: &mut Self::RwTxn<'a>, edge_id: &u128) -> Result<(), GraphError>; + fn update_node<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + id: &u128, + properties: &Value, + ) -> Result; + fn update_edge<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + id: &u128, + properties: &Value, + ) -> Result; + + fn get_out_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + ) -> Result, GraphError>; + + fn get_in_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + to_id: &u128, + ) -> Result, GraphError>; + + fn get_out_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + ) -> Result, GraphError>; + + fn get_in_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + to_id: &u128, + ) -> Result, GraphError>; + + fn shortest_path<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + to_id: &u128, + ) -> Result<(Vec, Vec), GraphError>; + + fn shortest_mutual_path<'a>( + &self, + txn: &Self::RoTxn<'a>, + edge_label: &str, + from_id: &u128, + to_id: &u128, + ) -> Result<(Vec, Vec), GraphError>; + + fn add_edge<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + label: &str, + properties: Option>, + from_node: u128, + to_node: u128, + ) -> Result; + + fn node_from_index<'a, K>( + &self, + txn: &Self::RoTxn<'a>, + index: &str, + key: K, + ) -> Result, GraphError> + where + K: Into + Serialize; + + fn add_node<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + label: &str, + properties: Option>, + secondary_indices: Option<&[&str]>, + ) -> Result; + + fn get_all_nodes<'a>( + &self, + txn: &Self::RoTxn<'a>, + ) -> Result> + 'a>, GraphError>; + + fn get_all_edges<'a>( + &self, + txn: &Self::RoTxn<'a>, + ) -> Result> + 'a>, GraphError>; + + fn index_node<'a, K>( + &self, + txn: &mut Self::RwTxn<'a>, + index: &str, + key: K, + node: &Node, + ) -> Result<(), GraphError> + where + K: Into + Serialize; +} \ No newline at end of file diff --git a/helixdb/src/lib.rs b/helixdb/src/lib.rs index eea321429..998825b60 100644 --- a/helixdb/src/lib.rs +++ b/helixdb/src/lib.rs @@ -6,4 +6,5 @@ pub mod helixc; pub mod ingestion_engine; pub mod protocol; pub mod helix_runtime; +pub mod helix_storage; pub mod helix_transport; From ccfe66b78bfc9aa48f11e6d9544829852567f154 Mon Sep 17 00:00:00 2001 From: skep Date: Mon, 9 Jun 2025 08:08:47 +0000 Subject: [PATCH 3/3] WIP: finish Lmbd_storage --- Cargo.lock | 1324 ++++++++++------- helixdb/Cargo.toml | 2 +- .../helix_engine/graph_core/ops/in_/in_.rs | 1 - .../helix_engine/graph_core/ops/out/out.rs | 1 - .../ops/vectors/brute_force_search.rs | 4 +- .../graph_core/ops/vectors/search.rs | 5 +- .../helix_engine/vector_core/vector_core.rs | 1 + helixdb/src/helix_gateway/mcp/mcp.rs | 3 +- helixdb/src/helix_storage/lmdb_storage.rs | 123 +- helixdb/src/helix_storage/mod.rs | 63 +- 10 files changed, 943 insertions(+), 584 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59aeb46c7..d0c31ffc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,22 +33,22 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "once_cell", "version_check", ] [[package]] name = "ahash" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.2.15", + "getrandom 0.3.3", "once_cell", "version_check", - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -113,59 +113,59 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "argminmax" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52424b59d69d69d5056d508b260553afd91c57e21849579cd1f50ee8b8b88eaa" +checksum = "70f13d10a41ac8d2ec79ee34178d61e6f47a29c2edfe7ef1721c7383b0359e65" dependencies = [ "num-traits", ] [[package]] name = "array-init-cursor" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" +checksum = "ed51fe0f224d1d4ea768be38c51f9f831dee9d05c163c11fba0b8c44387b1fc3" [[package]] name = "arrayvec" @@ -173,6 +173,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii-canvas" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6" +dependencies = [ + "term", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -192,25 +201,28 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "async-trait" -version = "0.1.87" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d556ec1359574147ec0c4fc5eb525f3f23263a592b1a9c07e0a75b427de55c97" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "atoi_simd" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4790f9e8961209112beb783d85449b508673cf4a6a419c8449b210743ac4dbe9" +checksum = "c2a49e05797ca52e312a0c658938b7d00693ef037799ef7187678f212d7684cf" +dependencies = [ + "debug_unsafe", +] [[package]] name = "atomic" @@ -235,9 +247,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.6.1" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" +checksum = "02a18fd934af6ae7ca52410d4548b98eb895aab0f1ea417d168d85db1434a141" dependencies = [ "aws-credential-types", "aws-runtime", @@ -265,9 +277,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" +checksum = "687bc16bc431a8533fe0097c7f0182874767f920989d7260950172ae8e3c4465" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -277,9 +289,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" dependencies = [ "aws-lc-sys", "zeroize", @@ -287,9 +299,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ "bindgen", "cc", @@ -300,9 +312,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" +checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -317,7 +329,6 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -326,9 +337,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.82.0" +version = "1.91.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6eab2900764411ab01c8e91a76fd11a63b4e12bc3da97d9e14a0ce1343d86d3" +checksum = "10c7d58f9c99e7d33e5a9b288ec84db24de046add7ba4c1e98baf6b3a5b37fde" dependencies = [ "aws-credential-types", "aws-runtime", @@ -351,7 +362,6 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "lru", - "once_cell", "percent-encoding", "regex-lite", "sha2", @@ -361,9 +371,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.64.0" +version = "1.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" +checksum = "13118ad30741222f67b1a18e5071385863914da05124652b38e172d6d3d9ce31" dependencies = [ "aws-credential-types", "aws-runtime", @@ -377,16 +387,15 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "once_cell", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-ssooidc" -version = "1.65.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" +checksum = "f879a8572b4683a8f84f781695bebf2f25cf11a81a2693c31fc0e0215c2c1726" dependencies = [ "aws-credential-types", "aws-runtime", @@ -400,16 +409,15 @@ dependencies = [ "bytes", "fastrand", "http 0.2.12", - "once_cell", "regex-lite", "tracing", ] [[package]] name = "aws-sdk-sts" -version = "1.65.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" +checksum = "f1e9c3c24e36183e2f698235ed38dcfbbdff1d09b9232dc866c4be3011e0b47e" dependencies = [ "aws-credential-types", "aws-runtime", @@ -424,16 +432,15 @@ dependencies = [ "aws-types", "fastrand", "http 0.2.12", - "once_cell", "regex-lite", "tracing", ] [[package]] name = "aws-sigv4" -version = "1.3.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" +checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -447,7 +454,6 @@ dependencies = [ "hmac", "http 0.2.12", "http 1.3.1", - "once_cell", "p256", "percent-encoding", "ring", @@ -471,16 +477,14 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.63.1" +version = "0.63.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65d21e1ba6f2cdec92044f904356a19f5ad86961acf015741106cdfafd747c0" +checksum = "d2f77a921dbd2c78ebe70726799787c1d110a2245dd65e39b20923dfdfb2deee" dependencies = [ "aws-smithy-http", "aws-smithy-types", "bytes", - "crc32c", - "crc32fast", - "crc64fast-nvme", + "crc-fast", "hex", "http 0.2.12", "http-body 0.4.6", @@ -504,9 +508,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.62.0" +version = "0.62.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" +checksum = "99335bec6cdc50a346fda1437f9fefe33abf8c99060739a546a16457f2862ca9" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -517,7 +521,6 @@ dependencies = [ "http 0.2.12", "http 1.3.1", "http-body 0.4.6", - "once_cell", "percent-encoding", "pin-project-lite", "pin-utils", @@ -526,25 +529,26 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" +checksum = "073d330f94bdf1f47bb3e0f5d45dda1e372a54a553c39ab6e9646902c8c81594" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "h2 0.4.8", + "h2 0.3.26", + "h2 0.4.10", "http 0.2.12", "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", "hyper 1.6.0", "hyper-rustls 0.24.2", - "hyper-rustls 0.27.5", + "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.25", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -563,12 +567,11 @@ dependencies = [ [[package]] name = "aws-smithy-observability" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445d065e76bc1ef54963db400319f1dd3ebb3e0a74af20f7f7630625b0cc7cc0" +checksum = "9364d5989ac4dd918e5cc4c4bdcc61c9be17dcd2586ea7f69e348fc7c6cab393" dependencies = [ "aws-smithy-runtime-api", - "once_cell", ] [[package]] @@ -583,9 +586,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.8.1" +version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" +checksum = "14302f06d1d5b7d333fd819943075b13d27c7700b414f574c3c35859bfb55d5e" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -599,7 +602,6 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", - "once_cell", "pin-project-lite", "pin-utils", "tokio", @@ -608,9 +610,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.7.4" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" +checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -625,9 +627,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" +checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" dependencies = [ "base64-simd", "bytes", @@ -660,9 +662,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.6" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" +checksum = "8a322fec39e4df22777ed3ad8ea868ac2f94cd15e1a55f6ee8d8d6305057689a" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -674,9 +676,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -717,9 +719,9 @@ dependencies = [ [[package]] name = "base64ct" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bincode" @@ -736,7 +738,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -749,10 +751,25 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.98", + "syn 2.0.101", "which", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -761,9 +778,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" dependencies = [ "serde", ] @@ -809,7 +826,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -825,9 +842,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -835,9 +852,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "bytecheck" @@ -863,22 +880,22 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.8.1" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fa76293b4f7bb636ab88fd78228235b5248b4d05cc589aed610f954af5d7c7a" +checksum = "7ecc273b49b3205b83d648f0690daa588925572cc5063745bfe547fe7ec8e1a1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -889,9 +906,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" dependencies = [ "serde", ] @@ -917,9 +934,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.19" +version = "1.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" +checksum = "956a5e21988b87f372569b66183b78babf23ebc2e744b733e4350a752c4dafac" dependencies = [ "jobserver", "libc", @@ -996,9 +1013,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.37" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" +checksum = "fd60e63e9be68e5fb56422e397cf9baddded06dae1d2e523401542383bc72a9f" dependencies = [ "clap_builder", "clap_derive", @@ -1006,9 +1023,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.37" +version = "4.5.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" +checksum = "89cc6392a1f72bbeb820d71f32108f61fdaf18bc526e1d23954168a67759ef51" dependencies = [ "anstream", "anstyle", @@ -1025,7 +1042,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -1045,9 +1062,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "colored" @@ -1116,9 +1133,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -1141,9 +1158,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -1155,12 +1172,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] -name = "crc32c" -version = "0.6.8" +name = "crc-fast" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +checksum = "68fcb2be5386ffb77e30bf10820934cb89a628bcb976e7cc632dcd88c059ebea" dependencies = [ - "rustc_version", + "cc", + "crc", + "digest", + "libc", + "rand 0.9.1", + "regex", ] [[package]] @@ -1172,20 +1194,11 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crc64fast-nvme" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4955638f00a809894c947f85a024020a20815b65a5eea633798ea7924edab2b3" -dependencies = [ - "crc", -] - [[package]] name = "crossbeam-channel" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] @@ -1230,7 +1243,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "crossterm_winapi", "parking_lot", "rustix 0.38.44", @@ -1246,6 +1259,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "crypto-bigint" version = "0.4.9" @@ -1278,6 +1297,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "debug_unsafe" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d3cef41d236720ed453e102153a53e4cc3d2fde848c0078a50cf249e8e3e5b" + [[package]] name = "der" version = "0.6.1" @@ -1290,9 +1315,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] @@ -1326,6 +1351,16 @@ dependencies = [ "dirs-sys 0.5.0", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + [[package]] name = "dirs-sys" version = "0.4.1" @@ -1350,6 +1385,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users 0.4.6", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1358,7 +1404,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -1396,9 +1442,9 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "elliptic-curve" @@ -1420,6 +1466,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ena" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d248bdd43ce613d87415282f69b9bb99d947d290b10962dd6c56233312c2ad5" +dependencies = [ + "log", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -1444,7 +1499,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -1455,9 +1510,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "cea14ef9355e3beab063703aa9dab15afd25f0667c341310c1e5274bb1d0da18" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1465,9 +1520,9 @@ dependencies = [ [[package]] name = "ethnum" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90ca2580b73ab6a1f724b76ca11ab632df820fd6040c336200d2c1df7b3c82c" +checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" [[package]] name = "fallible-iterator" @@ -1501,9 +1556,9 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "faststr" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403ebc0cd0c6dbff1cae7098168eff6bac83fad5928b6e91f29388b8dbb61653" +checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c" dependencies = [ "bytes", "rkyv 0.8.10", @@ -1521,11 +1576,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -1657,7 +1718,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -1707,14 +1768,14 @@ version = "0.1.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -1725,14 +1786,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1779,9 +1840,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -1821,7 +1882,7 @@ version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "allocator-api2", "rayon", "serde", @@ -1829,9 +1890,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -1846,7 +1907,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.4", ] [[package]] @@ -1860,7 +1921,7 @@ dependencies = [ "helixdb", "serde", "socket2", - "sonic-rs", + "sonic-rs 0.5.1", "tokio", ] @@ -1902,7 +1963,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8cb8e15e526c34dc6aa19ee62c125324e34af32bcfe4cb8b0de4be621735f007" dependencies = [ "aead", - "bitflags 2.8.0", + "bitflags 2.9.1", "byteorder", "generic-array", "heed-traits", @@ -1930,7 +1991,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sonic-rs", + "sonic-rs 0.5.1", "spinners", "tempfile", "tokio", @@ -1952,7 +2013,7 @@ dependencies = [ "rand 0.9.1", "serde", "serde_json", - "sonic-rs", + "sonic-rs 0.5.1", "tokio", "uuid", ] @@ -1962,6 +2023,7 @@ name = "helixdb" version = "1.0.85" dependencies = [ "bincode", + "bytes", "chrono", "colored", "dirs 6.0.0", @@ -1971,6 +2033,7 @@ dependencies = [ "inventory", "itertools 0.14.0", "kdam", + "lalrpop", "lazy_static", "native-tls", "pest", @@ -1985,8 +2048,9 @@ dependencies = [ "rust_decimal", "serde", "serde_json", - "sonic-rs", + "sonic-rs 0.3.17", "tempfile", + "thiserror 1.0.69", "tokio", "tokio-postgres", "twox-hash", @@ -2124,7 +2188,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "httparse", @@ -2153,15 +2217,14 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", "http 1.3.1", "hyper 1.6.0", "hyper-util", - "rustls 0.23.25", + "rustls 0.23.27", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -2187,35 +2250,43 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", + "ipnet", + "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core 0.61.2", ] [[package]] @@ -2229,21 +2300,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -2252,31 +2324,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -2284,67 +2336,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", -] - [[package]] name = "idna" version = "1.0.3" @@ -2358,9 +2397,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -2368,12 +2407,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "serde", ] @@ -2392,9 +2431,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b12ebb6799019b044deaf431eadfe23245b259bba5a2c0796acec3943a3cdb" +checksum = "ab08d7cd2c5897f2c949e5383ea7c7db03fb19130ffcfbf7eda795137ae3cb83" dependencies = [ "rustversion", ] @@ -2416,6 +2455,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-docker" version = "0.2.0" @@ -2441,6 +2490,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2461,16 +2519,17 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] @@ -2505,6 +2564,37 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "lalrpop" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" +dependencies = [ + "ascii-canvas", + "bit-set", + "ena", + "itertools 0.11.0", + "lalrpop-util", + "petgraph", + "pico-args", + "regex", + "regex-syntax", + "string_cache", + "term", + "tiny-keccak", + "unicode-xid", + "walkdir", +] + +[[package]] +name = "lalrpop-util" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +dependencies = [ + "regex-automata", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2525,19 +2615,19 @@ checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] name = "libm" -version = "0.2.11" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" @@ -2545,7 +2635,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "libc", ] @@ -2562,9 +2652,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -2589,9 +2679,9 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lmdb-master3-sys" @@ -2606,9 +2696,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -2616,9 +2706,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "lru" @@ -2626,7 +2716,7 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" dependencies = [ - "hashbrown 0.15.2", + "hashbrown 0.15.4", ] [[package]] @@ -2693,42 +2783,42 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "munge" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64142d38c84badf60abf06ff9bd80ad2174306a5b11bd4706535090a30a419df" +checksum = "9e22e7961c873e8b305b176d2a4e1d41ce7ba31bc1c52d2a107a89568ec74c55" dependencies = [ "munge_macro", ] [[package]] name = "munge_macro" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bb5c1d8184f13f7d0ccbeeca0def2f9a181bce2624302793005f5ca8aa62e5e" +checksum = "0ac7d860b767c6398e88fe93db73ce53eb496057aa6895ffa4d60cb02e1d1c6b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -2737,7 +2827,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -2757,6 +2847,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + [[package]] name = "nom" version = "7.1.3" @@ -2827,9 +2923,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" [[package]] name = "open" @@ -2844,11 +2946,11 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.72" +version = "0.10.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cfg-if", "foreign-types", "libc", @@ -2865,7 +2967,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -2876,9 +2978,9 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-sys" -version = "0.9.107" +version = "0.9.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" dependencies = [ "cc", "libc", @@ -2921,9 +3023,9 @@ dependencies = [ [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -2931,9 +3033,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -2970,7 +3072,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "198db74531d58c70a361c42201efde7e2591e976d518caf7662a47dc5720e7b6" dependencies = [ "memchr", - "thiserror 2.0.11", + "thiserror 2.0.12", "ucd-trie", ] @@ -2994,7 +3096,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -3008,11 +3110,21 @@ dependencies = [ "sha2", ] +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pgvector" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0e8871b6d7ca78348c6cd29b911b94851f3429f0cd403130ca17f26c1fb91a6" +checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b" dependencies = [ "bytes", "postgres-types", @@ -3058,7 +3170,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -3070,6 +3182,12 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pico-args" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be167a7af36ee22fe3115051bc51f6e6c7054c9348e28deb4f49bd6f705a315" + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -3113,7 +3231,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72571dde488ecccbe799798bf99ab7308ebdb7cf5d95bcc498dbd5a132f0da4d" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "polars-arrow", "polars-core", "polars-error", @@ -3133,7 +3251,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6611c758d52e799761cc25900666b71552e6c929d88052811bc9daad4b3321a8" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "atoi_simd", "bytemuck", "chrono", @@ -3141,8 +3259,8 @@ dependencies = [ "dyn-clone", "either", "ethnum", - "getrandom 0.2.15", - "hashbrown 0.15.2", + "getrandom 0.2.16", + "hashbrown 0.15.4", "itoa", "lz4", "num-traits", @@ -3196,15 +3314,15 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "796d06eae7e6e74ed28ea54a8fccc584ebac84e6cf0e1e9ba41ffc807b169a01" dependencies = [ - "ahash 0.8.11", - "bitflags 2.8.0", + "ahash 0.8.12", + "bitflags 2.9.1", "bytemuck", "chrono", "chrono-tz", "comfy-table", "either", "hashbrown 0.14.5", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "indexmap", "itoa", "num-traits", @@ -3220,7 +3338,7 @@ dependencies = [ "rayon", "regex", "strum_macros 0.26.4", - "thiserror 2.0.11", + "thiserror 2.0.12", "version_check", "xxhash-rust", ] @@ -3234,7 +3352,7 @@ dependencies = [ "polars-arrow-format", "regex", "simdutf8", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -3243,9 +3361,9 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8e639991a8ad4fb12880ab44bcc3cf44a5703df003142334d9caf86d77d77e7" dependencies = [ - "ahash 0.8.11", - "bitflags 2.8.0", - "hashbrown 0.15.2", + "ahash 0.8.12", + "bitflags 2.9.1", + "hashbrown 0.15.4", "num-traits", "once_cell", "polars-arrow", @@ -3267,7 +3385,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719a77e94480f6be090512da196e378cbcbeb3584c6fe1134c600aee906e38ab" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "async-trait", "atoi_simd", "bytes", @@ -3275,7 +3393,7 @@ dependencies = [ "fast-float2", "futures", "glob", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "home", "itoa", "memchr", @@ -3306,10 +3424,10 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e30603ca81e317b66b4caac683a8325a6a82ea0489685dc37e22ae03720def98" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "chrono", "fallible-streaming-iterator", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "indexmap", "itoa", "num-traits", @@ -3328,8 +3446,8 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0a731a672dfc8ac38c1f73c9a4b2ae38d2fc8ac363bfb64c5f3a3e072ffc5ad" dependencies = [ - "ahash 0.8.11", - "bitflags 2.8.0", + "ahash 0.8.12", + "bitflags 2.9.1", "chrono", "memchr", "once_cell", @@ -3375,14 +3493,14 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbb83218b0c216104f0076cd1a005128be078f958125f3d59b094ee73d78c18e" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "argminmax", "base64 0.22.1", "bytemuck", "chrono", "chrono-tz", "either", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "hex", "indexmap", "jsonpath_lib_polars_vendor", @@ -3412,7 +3530,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c60ee85535590a38db6c703a21be4cb25342e40f573f070d1e16f9d84a53ac7" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "async-stream", "base64 0.22.1", "brotli", @@ -3420,7 +3538,7 @@ dependencies = [ "ethnum", "flate2", "futures", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "lz4", "num-traits", "polars-arrow", @@ -3454,7 +3572,7 @@ dependencies = [ "crossbeam-queue", "enum_dispatch", "futures", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "num-traits", "once_cell", "polars-arrow", @@ -3477,14 +3595,14 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f03533a93aa66127fcb909a87153a3c7cfee6f0ae59f497e73d7736208da54c" dependencies = [ - "ahash 0.8.11", - "bitflags 2.8.0", + "ahash 0.8.12", + "bitflags 2.9.1", "bytemuck", "bytes", "chrono", "chrono-tz", "either", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "memmap2", "num-traits", "once_cell", @@ -3511,7 +3629,7 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bf47f7409f8e75328d7d034be390842924eb276716d0458607be0bddb8cc839" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "bytemuck", "polars-arrow", "polars-compute", @@ -3611,11 +3729,11 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f6c8166a4a7fbc15b87c81645ed9e1f0651ff2e8c96cafc40ac5bf43441a10" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "bytemuck", "bytes", "compact_str", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "indexmap", "libc", "memmap2", @@ -3632,9 +3750,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "postgres-native-tls" @@ -3679,6 +3797,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3687,21 +3814,27 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.7.35", + "zerocopy", ] +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "9dee91521343f4c5c6a63edd65e54f31f5c92fe8978c40a4282f8372194c6a7d" dependencies = [ "proc-macro2", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -3715,18 +3848,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] [[package]] name = "psm" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" +checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" dependencies = [ "cc", ] @@ -3768,18 +3901,24 @@ checksum = "ca414edb151b4c8d125c12566ab0d74dc9cdba36fb80eb7b848c15f495fd32d1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -3813,7 +3952,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", + "rand_core 0.9.3", ] [[package]] @@ -3833,7 +3972,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.3", ] [[package]] @@ -3842,17 +3981,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", - "zerocopy 0.8.19", + "getrandom 0.3.3", ] [[package]] @@ -3871,7 +4009,7 @@ version = "11.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] @@ -3911,16 +4049,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] @@ -3929,7 +4067,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "libredox", "thiserror 1.0.69", ] @@ -3940,29 +4078,29 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", "libredox", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] name = "ref-cast" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4017,9 +4155,9 @@ checksum = "a35e8a6bf28cd121053a66aa2e6a2e3eaffad4a60012179f0e864aa5ffeff215" [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "a2f8e5513d63f2e5b386eb5106dc67eaf3f84e95258e210489136b8b92ad6119" dependencies = [ "base64 0.22.1", "bytes", @@ -4027,12 +4165,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.8", + "h2 0.4.10", "http 1.3.1", "http-body 1.0.1", "http-body-util", "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper-rustls 0.27.7", "hyper-tls", "hyper-util", "ipnet", @@ -4043,21 +4181,20 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile 2.2.0", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", - "system-configuration", "tokio", "tokio-native-tls", "tower", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows-registry", ] [[package]] @@ -4079,7 +4216,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -4110,7 +4247,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e147371c75553e1e2fcdb483944a8540b8438c31426279553b9a8182a9b7b65" dependencies = [ "bytes", - "hashbrown 0.15.2", + "hashbrown 0.15.4", "indexmap", "munge", "ptr_meta 0.3.0", @@ -4140,7 +4277,7 @@ checksum = "246b40ac189af6c675d124b802e8ef6d5246c53e17367ce9501f8f66a81abb7a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4149,7 +4286,7 @@ version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a22715a5d6deef63c637207afbe68d0c72c3f8d0022d7cf9714c442d6157606b" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", @@ -4215,7 +4352,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4224,11 +4361,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.5" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", @@ -4249,14 +4386,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "once_cell", "rustls-pki-types", - "rustls-webpki 0.103.1", + "rustls-webpki 0.103.3", "subtle", "zeroize", ] @@ -4268,7 +4405,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", - "rustls-pemfile 1.0.4", + "rustls-pemfile", "schannel", "security-framework 2.11.1", ] @@ -4295,20 +4432,14 @@ dependencies = [ ] [[package]] -name = "rustls-pemfile" -version = "2.2.0" +name = "rustls-pki-types" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ - "rustls-pki-types", + "zeroize", ] -[[package]] -name = "rustls-pki-types" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4321,9 +4452,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "aws-lc-rs", "ring", @@ -4333,15 +4464,24 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.19" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "ryu" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] [[package]] name = "schannel" @@ -4394,7 +4534,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4407,8 +4547,8 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", - "core-foundation 0.10.0", + "bitflags 2.9.1", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4447,7 +4587,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4465,9 +4605,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -4497,9 +4637,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -4514,9 +4654,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -4537,8 +4677,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2bcf6c6e164e81bc7a5d49fc6988b3d515d9e8c07457d7b74ffb9324b9cd40" dependencies = [ - "ahash 0.8.11", - "getrandom 0.2.15", + "ahash 0.8.12", + "getrandom 0.2.16", "halfbrown", "once_cell", "ref-cast", @@ -4580,9 +4720,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "snap" @@ -4592,9 +4732,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4609,13 +4749,33 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "sonic-rs" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0275f9f2f07d47556fe60c2759da8bc4be6083b047b491b2d476aa0bfa558eb1" +dependencies = [ + "bumpalo", + "bytes", + "cfg-if", + "faststr", + "itoa", + "ref-cast", + "ryu", + "serde", + "simdutf8", + "sonic-number", + "sonic-simd", + "thiserror 2.0.12", +] + [[package]] name = "sonic-rs" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93070f7e7c0d7ec7d08406b1b407234af30420320fd854f304029e3c6db4a899" dependencies = [ - "ahash 0.8.11", + "ahash 0.8.12", "bumpalo", "bytes", "cfg-if", @@ -4627,14 +4787,14 @@ dependencies = [ "simdutf8", "sonic-number", "sonic-simd", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] name = "sonic-simd" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940a24e82c9a97483ef66cef06b92160a8fa5cd74042c57c10b24d99d169d2fc" +checksum = "b421f7b6aa4a5de8f685aaf398dfaa828346ee639d2b1c1061ab43d40baa6223" dependencies = [ "cfg-if", ] @@ -4686,9 +4846,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601f9201feb9b09c00266478bf459952b9ef9a6b94edb2f21eba14ab681a60a9" +checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" dependencies = [ "cc", "cfg-if", @@ -4724,6 +4884,18 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" +[[package]] +name = "string_cache" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" +dependencies = [ + "new_debug_unreachable", + "parking_lot", + "phf_shared", + "precomputed-hash", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -4773,7 +4945,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4795,9 +4967,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -4824,13 +4996,13 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4852,7 +5024,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -4875,17 +5047,28 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.19.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.1", + "getrandom 0.3.3", "once_cell", - "rustix 1.0.5", + "rustix 1.0.7", "windows-sys 0.59.0", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "terminal_size" version = "0.2.6" @@ -4907,11 +5090,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -4922,18 +5105,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -4966,11 +5149,20 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -4978,9 +5170,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -4993,9 +5185,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -5017,7 +5209,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -5072,15 +5264,15 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.25", + "rustls 0.23.27", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -5091,9 +5283,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.20" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -5103,26 +5295,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.2" @@ -5138,6 +5337,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags 2.9.1", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -5163,20 +5380,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "1b1ffbcf9c6f6b99d386e7444eb608ba646ae452a36b39737deb9663b610f662" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", ] @@ -5216,9 +5433,9 @@ checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -5256,6 +5473,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "untrusted" version = "0.9.0" @@ -5279,12 +5502,6 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5299,13 +5516,15 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "atomic", - "getrandom 0.3.1", + "getrandom 0.3.3", + "js-sys", "rand 0.9.1", + "wasm-bindgen", ] [[package]] @@ -5338,6 +5557,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -5355,9 +5584,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -5390,7 +5619,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -5425,7 +5654,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5498,6 +5727,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -5521,7 +5759,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" dependencies = [ "windows-collections", - "windows-core 0.61.0", + "windows-core 0.61.2", "windows-future", "windows-link", "windows-numerics", @@ -5533,16 +5771,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-core 0.61.0", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.6", + "windows-core 0.61.2", ] [[package]] @@ -5559,25 +5788,26 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.61.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement 0.60.0", "windows-interface 0.59.1", "windows-link", - "windows-result 0.3.2", - "windows-strings 0.4.0", + "windows-result 0.3.4", + "windows-strings", ] [[package]] name = "windows-future" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1d6bbefcb7b60acd19828e1bc965da6fcf18a7e39490c5f8be71e54a19ba32" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ - "windows-core 0.61.0", + "windows-core 0.61.2", "windows-link", + "windows-threading", ] [[package]] @@ -5588,7 +5818,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -5599,7 +5829,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -5610,7 +5840,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -5621,7 +5851,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] @@ -5636,19 +5866,19 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-core 0.61.0", + "windows-core 0.61.2", "windows-link", ] [[package]] name = "windows-registry" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "b3bab093bdd303a1240bb99b8aba8ea8a69ee19d34c9e2ef9594e708a4878820" dependencies = [ - "windows-result 0.3.2", - "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-link", + "windows-result 0.3.4", + "windows-strings", ] [[package]] @@ -5662,27 +5892,18 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.3.1" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ "windows-link", ] @@ -5761,6 +5982,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5901,33 +6131,27 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.6" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "wyz" @@ -5952,9 +6176,9 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -5964,75 +6188,54 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", "synstructure", ] [[package]] name = "zerocopy" -version = "0.7.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8207f485579465f62ae51a983e42c906736a17efd2de48b021e64f1bbd8e98c7" -dependencies = [ - "zerocopy-derive 0.8.19", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.98", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.19" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dbe1304a711c6eb4cf1ed333aa0d9b344685e71f6f00c3b176072213bd3783e" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", "synstructure", ] @@ -6042,11 +6245,22 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + [[package]] name = "zerovec" -version = "0.10.4" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -6055,20 +6269,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.101", ] [[package]] name = "zlib-rs" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" @@ -6081,18 +6295,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.3" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.14+zstd.1.5.7" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", "pkg-config", diff --git a/helixdb/Cargo.toml b/helixdb/Cargo.toml index 5639e22d6..37007bf95 100644 --- a/helixdb/Cargo.toml +++ b/helixdb/Cargo.toml @@ -12,7 +12,7 @@ tokio = { version = "1.44.2", features = ["full"] } serde = { version = "1.0.217", features = ["derive"] } serde_json = "1.0.110" bincode = "1.3.3" # TODO: Figure out bincode 2 impl with current serde impl -sonic-rs = { version = "0.3.1", features = ["base64-simd"] } +sonic-rs = { version = "0.3.1" } inventory = "0.3.16" twox-hash = "2.1.0" heed3 = "0.22.0" diff --git a/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs b/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs index 9cc32371b..81c5dd739 100644 --- a/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs +++ b/helixdb/src/helix_engine/graph_core/ops/in_/in_.rs @@ -8,7 +8,6 @@ use crate::{ }, helix_storage::Storage, }; -use heed3::{types::Bytes, RoTxn}; use std::sync::Arc; pub struct InNodesIterator<'a, I, S: Storage + ?Sized> { diff --git a/helixdb/src/helix_engine/graph_core/ops/out/out.rs b/helixdb/src/helix_engine/graph_core/ops/out/out.rs index f6889b687..0495524c9 100644 --- a/helixdb/src/helix_engine/graph_core/ops/out/out.rs +++ b/helixdb/src/helix_engine/graph_core/ops/out/out.rs @@ -11,7 +11,6 @@ use crate::{ }, helix_storage::Storage, }; -use heed3::{types::Bytes, RoTxn, WithTls}; use std::sync::Arc; pub struct OutNodesIterator<'a, I, S: Storage + ?Sized> { diff --git a/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs b/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs index a84b5d8dc..0ef985f6b 100644 --- a/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs +++ b/helixdb/src/helix_engine/graph_core/ops/vectors/brute_force_search.rs @@ -1,5 +1,3 @@ -use heed3::RoTxn; - use super::super::tr_val::TraversalVal; use crate::helix_engine::{ graph_core::traversal_iter::RoTraversalIterator, @@ -9,8 +7,8 @@ use crate::helix_engine::{ vector::{cosine_similarity, HVector}, }, }; +use crate::helix_storage::{lmdb_storage::LmdbStorage, Storage}; use std::{collections::BinaryHeap, iter::once}; -use crate::helix_storage::Storage; pub struct BruteForceSearchV>> { iter: I, diff --git a/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs b/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs index 2926d80e9..d21630a10 100644 --- a/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs +++ b/helixdb/src/helix_engine/graph_core/ops/vectors/search.rs @@ -5,7 +5,6 @@ use crate::helix_engine::{ vector_core::{hnsw::HNSW, vector::HVector}, }; use crate::helix_storage::{lmdb_storage::LmdbStorage, Storage}; -use heed3::RoTxn; use std::iter::once; pub struct SearchV>> { @@ -30,7 +29,7 @@ pub trait SearchVAdapter<'a, S: Storage + ?Sized>: filter: Option<&[F]>, ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&HVector, &heed3::RoTxn<'a>) -> bool; + F: Fn(&HVector, &S::RoTxn<'a>) -> bool; } impl<'a, I, S> SearchVAdapter<'a, S> for RoTraversalIterator<'a, I, S> @@ -45,7 +44,7 @@ where filter: Option<&[F]>, ) -> RoTraversalIterator<'a, impl Iterator>, S> where - F: Fn(&HVector, &heed3::RoTxn<'a>) -> bool, + F: Fn(&HVector, &S::RoTxn<'a>) -> bool, { let vectors = if let Some(lmdb_storage) = (self.storage.as_ref() as &dyn std::any::Any).downcast_ref::() diff --git a/helixdb/src/helix_engine/vector_core/vector_core.rs b/helixdb/src/helix_engine/vector_core/vector_core.rs index 867d6c312..066a869b0 100644 --- a/helixdb/src/helix_engine/vector_core/vector_core.rs +++ b/helixdb/src/helix_engine/vector_core/vector_core.rs @@ -3,6 +3,7 @@ use crate::helix_engine::{ vector_core::{hnsw::HNSW, vector::HVector}, }; use crate::protocol::value::Value; + use heed3::{ types::{Bytes, Unit}, Database, Env, RoTxn, RwTxn, diff --git a/helixdb/src/helix_gateway/mcp/mcp.rs b/helixdb/src/helix_gateway/mcp/mcp.rs index 96e05ca2a..5bee067f5 100644 --- a/helixdb/src/helix_gateway/mcp/mcp.rs +++ b/helixdb/src/helix_gateway/mcp/mcp.rs @@ -10,7 +10,6 @@ use std::{ }; use get_routes::{local_handler, mcp_handler}; -use heed3::{AnyTls, RoTxn}; use serde::Deserialize; use tokio::sync::mpsc; use tokio::sync::mpsc::{Receiver, Sender}; @@ -33,7 +32,7 @@ use crate::{ mcp::tools::{ToolArgs, ToolCalls}, router::router::HandlerInput, }, - helix_storage::lmdb_storage::LmdbStorage, + helix_storage::{lmdb_storage::LmdbStorage, Storage}, protocol::{ items::v6_uuid, label_hash::hash_label, request::Request, response::Response, return_values::ReturnValue, diff --git a/helixdb/src/helix_storage/lmdb_storage.rs b/helixdb/src/helix_storage/lmdb_storage.rs index 9ce395057..3b7ee83fb 100644 --- a/helixdb/src/helix_storage/lmdb_storage.rs +++ b/helixdb/src/helix_storage/lmdb_storage.rs @@ -16,6 +16,7 @@ use crate::helix_engine::types::GraphError; use crate::helix_engine::vector_core::hnsw::HNSW; use crate::helix_engine::vector_core::vector::HVector; use crate::helix_engine::vector_core::vector_core::{HNSWConfig, VectorCore}; +use crate::helix_engine::types::VectorError; use crate::protocol::items::{Edge, Node, v6_uuid}; use crate::protocol::label_hash::hash_label; use crate::protocol::value::Value; @@ -103,7 +104,7 @@ impl<'a> DerefMut for LmdbRwTxn<'a> { impl<'a, 'b> From<&'a mut LmdbRwTxn<'b>> for LmdbRoTxn<'a> { fn from(txn: &'a mut LmdbRwTxn<'b>) -> Self { - LmdbRoTxn(txn.0.deref()) + LmdbRoTxn(&txn.0) } } // endregion @@ -251,6 +252,14 @@ impl LmdbStorage { ); Ok((node_id, edge_id)) } + + /// Get a reference to the secondary index database by name + #[inline(always)] + pub fn get_node_index_db(&self, index: &str) -> Result<&Database>, GraphError> { + self.secondary_indices + .get(index) + .ok_or_else(|| GraphError::New(format!("Secondary Index {} not found", index))) + } } // endregion @@ -260,11 +269,11 @@ impl Storage for LmdbStorage { type RwTxn<'a> = LmdbRwTxn<'a>; fn ro_txn(&self) -> Result, GraphError> { - self.graph_env.read_txn().map(LmdbRoTxn).map_err(Into::into) + self.graph_env.read_txn().map(|txn| LmdbRoTxn(txn)).map_err(Into::into) } fn rw_txn(&self) -> Result, GraphError> { - self.graph_env.write_txn().map(LmdbRwTxn).map_err(Into::into) + self.graph_env.write_txn().map(|txn| LmdbRwTxn(txn)).map_err(Into::into) } fn create_secondary_index(&mut self, name: &str) -> Result<(), GraphError> { @@ -355,11 +364,11 @@ impl Storage for LmdbStorage { } fn drop_edge<'a>(&self, txn: &mut Self::RwTxn<'a>, edge_id: &u128) -> Result<(), GraphError> { - let edge_data = self + let mut edge = self .edges_db .get(txn, Self::edge_key(edge_id))? - .ok_or(GraphError::EdgeNotFound)?; - let edge: Edge = bincode::deserialize(edge_data)?; + .ok_or_else(|| GraphError::EdgeNotFound(*edge_id))?; + edge.id = *edge_id; let label_hash = hash_label(&edge.label, None); self.edges_db.delete(txn, Self::edge_key(edge_id))?; @@ -377,7 +386,7 @@ impl Storage for LmdbStorage { id: &u128, properties: &Value, ) -> Result { - let mut old_node = self.get_node(&*txn, id)?; + let mut old_node = self.get_node(&LmdbRoTxn::from(txn), id)?; if let Value::Object(props) = properties { old_node.properties = Some(props.clone()); } @@ -391,7 +400,7 @@ impl Storage for LmdbStorage { id: &u128, properties: &Value, ) -> Result { - let mut old_edge = self.get_edge(&*txn, id)?; + let mut old_edge = self.get_edge(&LmdbRoTxn::from(txn), id)?; if let Value::Object(props) = properties { old_edge.properties = Some(props.clone()); } @@ -417,9 +426,8 @@ impl Storage for LmdbStorage { to_node, }; - let bytes = edge.encode_edge()?; self.edges_db - .put(txn, &Self::edge_key(&edge.id), &bytes)?; + .put(txn, &Self::edge_key(&edge.id), &edge)?; let label_hash = hash_label(edge.label.as_str(), None); @@ -588,10 +596,7 @@ impl Storage for LmdbStorage { let result = db.get(txn, &key_bytes)?; - if let Some(node_id_bytes) = result { - let node_id = u128::from_be_bytes(node_id_bytes.try_into().map_err(|_| { - GraphError::ConversionError("Invalid node ID length in index".to_string()) - })?); + if let Some(node_id) = result { return self.get_node(txn, &node_id).map(Some); } @@ -627,8 +632,7 @@ impl Storage for LmdbStorage { properties: properties.map(|props| props.into_iter().collect()), }; - let bytes = node.encode_node()?; - self.nodes_db.put(txn, &node.id, &bytes)?; + self.nodes_db.put(txn, &node.id, &node)?; if let Some(indices) = secondary_indices { for index_name in indices { @@ -638,7 +642,7 @@ impl Storage for LmdbStorage { if let Some(value) = node.properties.as_ref().and_then(|p| p.get(*index_name)) { let key_bytes = bincode::serialize(value)?; - db.put(txn, &key_bytes, &node.id.to_be_bytes())?; + db.put(txn, &key_bytes, &node.id)?; } } } @@ -678,5 +682,90 @@ impl Storage for LmdbStorage { .map_err(GraphError::from) }))) } + + // Vector operations implementation + fn search_vectors<'a, F>( + &self, + txn: &Self::RoTxn<'a>, + query: &[f64], + k: usize, + filter: Option<&[F]>, + ) -> Result, VectorError> + where + F: Fn(&HVector, &Self::RoTxn<'a>) -> bool, + { + // Convert our transaction to the HEED transaction that HNSW expects + self.vectors.search(&txn.0, query, k, filter, false) + } + + fn insert_vector<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + data: &[f64], + fields: Option>, + ) -> Result { + // Convert our transaction to the HEED transaction that HNSW expects + self.vectors.insert(&mut txn.0, data, fields) + } + + fn get_vector<'a>( + &self, + txn: &Self::RoTxn<'a>, + id: u128, + level: usize, + with_data: bool, + ) -> Result { + // Convert our transaction to the HEED transaction that HNSW expects + self.vectors.get_vector(&txn.0, id, level, with_data) + } + + fn get_all_vectors<'a>( + &self, + txn: &Self::RoTxn<'a>, + level: Option, + ) -> Result, VectorError> { + // Convert our transaction to the HEED transaction that HNSW expects + self.vectors.get_all_vectors(&txn.0, level) + } + + // BM25 operations implementation + fn insert_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + doc: &str, + ) -> Result<(), GraphError> { + // Convert our transaction to the HEED transaction that BM25 expects + self.bm25.insert_doc(&mut txn.0, doc_id, doc) + } + + fn update_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + doc: &str, + ) -> Result<(), GraphError> { + // Convert our transaction to the HEED transaction that BM25 expects + self.bm25.update_doc(&mut txn.0, doc_id, doc) + } + + fn delete_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + ) -> Result<(), GraphError> { + // Convert our transaction to the HEED transaction that BM25 expects + self.bm25.delete_doc(&mut txn.0, doc_id) + } + + fn search_bm25<'a>( + &self, + txn: &Self::RoTxn<'a>, + query: &str, + limit: usize, + ) -> Result, GraphError> { + // Convert our transaction to the HEED transaction that BM25 expects + self.bm25.search(&txn.0, query, limit) + } } // endregion \ No newline at end of file diff --git a/helixdb/src/helix_storage/mod.rs b/helixdb/src/helix_storage/mod.rs index 2aa12f745..abd57dcb4 100644 --- a/helixdb/src/helix_storage/mod.rs +++ b/helixdb/src/helix_storage/mod.rs @@ -1,6 +1,7 @@ pub mod lmdb_storage; -use crate::helix_engine::types::GraphError; +use crate::helix_engine::types::{GraphError, VectorError}; +use crate::helix_engine::vector_core::vector::HVector; use crate::protocol::items::{Edge, Node}; use crate::protocol::value::Value; use serde::Serialize; @@ -128,4 +129,64 @@ pub trait Storage: Send + Sync + 'static { ) -> Result<(), GraphError> where K: Into + Serialize; + + // Vector operations + fn search_vectors<'a, F>( + &self, + txn: &Self::RoTxn<'a>, + query: &[f64], + k: usize, + filter: Option<&[F]>, + ) -> Result, VectorError> + where + F: Fn(&HVector, &Self::RoTxn<'a>) -> bool; + + fn insert_vector<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + data: &[f64], + fields: Option>, + ) -> Result; + + fn get_vector<'a>( + &self, + txn: &Self::RoTxn<'a>, + id: u128, + level: usize, + with_data: bool, + ) -> Result; + + fn get_all_vectors<'a>( + &self, + txn: &Self::RoTxn<'a>, + level: Option, + ) -> Result, VectorError>; + + // BM25 operations + fn insert_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + doc: &str, + ) -> Result<(), GraphError>; + + fn update_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + doc: &str, + ) -> Result<(), GraphError>; + + fn delete_bm25_doc<'a>( + &self, + txn: &mut Self::RwTxn<'a>, + doc_id: u128, + ) -> Result<(), GraphError>; + + fn search_bm25<'a>( + &self, + txn: &Self::RoTxn<'a>, + query: &str, + limit: usize, + ) -> Result, GraphError>; } \ No newline at end of file