From bf10be65ac8ab70b1aae697edbb35de28a7da590 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Fri, 29 May 2026 06:11:11 -0400 Subject: [PATCH 01/27] Current iteration. Persisted Vec, where entry i corresponds to global property id i corresponding to the segment. Persisted to disk. In-memory part grows incrementally. --- db4-storage/src/properties/mod.rs | 28 ++- db4-storage/src/segments/edge/segment.rs | 27 +++ db4-storage/src/segments/mod.rs | 8 + db4-storage/src/segments/node/segment.rs | 27 +++ .../core/entities/properties/layer_schema.rs | 163 ++++++++++++++++++ .../src/core/entities/properties/mod.rs | 1 + 6 files changed, 250 insertions(+), 4 deletions(-) create mode 100644 raphtory-api/src/core/entities/properties/layer_schema.rs diff --git a/db4-storage/src/properties/mod.rs b/db4-storage/src/properties/mod.rs index 0750490113..6c8647d66e 100644 --- a/db4-storage/src/properties/mod.rs +++ b/db4-storage/src/properties/mod.rs @@ -6,6 +6,7 @@ use arrow_array::{ use arrow_schema::DECIMAL128_MAX_PRECISION; use bigdecimal::ToPrimitive; use raphtory_api::core::entities::properties::{ + layer_schema::LayerPropSchema, meta::PropMapper, prop::{ AsPropRef, Prop, PropRef, PropType, SerdeArrowList, SerdeArrowMap, @@ -38,6 +39,10 @@ pub struct Properties { has_properties: bool, has_deletions: bool, pub additions_count: usize, + /// Per-segment record of which (global) property ids have ever been written + /// to this `Properties`. Maintained incrementally on every write so that + /// the layer schema can be derived without scanning entities. + layer_schema: LayerPropSchema, } pub(crate) struct PropMutEntry<'a> { @@ -94,6 +99,12 @@ impl Properties { self.t_properties.num_columns() } + /// Returns the per-segment property-presence summary maintained + /// incrementally as `append_t_props` / `append_const_props` are called. + pub fn layer_schema(&self) -> &LayerPropSchema { + &self.layer_schema + } + pub fn num_c_columns(&self) -> usize { self.c_properties.len() } @@ -281,10 +292,18 @@ impl<'a> PropMutEntry<'a> { t: EventTime, props: impl IntoIterator, ) { - let t_prop_row = if let Some(t_prop_row) = self - .properties - .t_properties - .push(props) + let Properties { + t_properties, + layer_schema, + .. + } = &mut *self.properties; + + let tracked = props + .into_iter() + .inspect(|(prop_id, _)| layer_schema.insert_temporal(*prop_id)); + + let t_prop_row = if let Some(t_prop_row) = t_properties + .push(tracked) .expect("Internal error: properties should be validated at this point") { t_prop_row @@ -353,6 +372,7 @@ impl<'a> PropMutEntry<'a> { let const_props = &mut self.properties.c_properties[prop_id]; // property types should have been validated before! const_props.upsert(self.row, prop.as_prop_ref()).unwrap(); + self.properties.layer_schema.insert_metadata(prop_id); } } } diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index 54e4d8ab0a..91e8aad566 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -133,6 +133,33 @@ impl MemEdgeSegment { self.layers.get(layer_id.0) } + /// Per-layer property-presence schema for this in-memory segment. Returns + /// an empty schema if the layer has no container in this segment. + pub fn layer_schema( + &self, + layer_id: LayerId, + ) -> raphtory_api::core::entities::properties::layer_schema::LayerPropSchema { + self.layers + .get(layer_id.0) + .map(|c| c.layer_schema().clone()) + .unwrap_or_default() + } + + /// Iterator over (layer_id, schema) for every layer present in this segment. + pub fn layer_schemas( + &self, + ) -> impl Iterator< + Item = ( + LayerId, + &raphtory_api::core::entities::properties::layer_schema::LayerPropSchema, + ), + > { + self.layers + .iter() + .enumerate() + .map(|(i, c)| (LayerId(i), c.layer_schema())) + } + pub fn est_size(&self) -> usize { self.est_size } diff --git a/db4-storage/src/segments/mod.rs b/db4-storage/src/segments/mod.rs index 222c8b2d71..e0451f7375 100644 --- a/db4-storage/src/segments/mod.rs +++ b/db4-storage/src/segments/mod.rs @@ -2,6 +2,7 @@ use super::properties::{PropEntry, Properties}; use crate::{LocalPOS, error::StorageError}; use raphtory_api::core::{ entities::properties::{ + layer_schema::LayerPropSchema, meta::Meta, prop::{AsPropRef, Prop}, }, @@ -250,6 +251,13 @@ impl SegmentContainer { &self.properties } + /// Per-layer property-presence summary for this container (one container = + /// one layer of one segment). Cheap — just borrows the incrementally + /// maintained bitsets. + pub fn layer_schema(&self) -> &LayerPropSchema { + self.properties.layer_schema() + } + pub fn properties_mut(&mut self) -> &mut Properties { &mut self.properties } diff --git a/db4-storage/src/segments/node/segment.rs b/db4-storage/src/segments/node/segment.rs index d556075f4e..90cfc78e8a 100644 --- a/db4-storage/src/segments/node/segment.rs +++ b/db4-storage/src/segments/node/segment.rs @@ -150,6 +150,33 @@ impl MemNodeSegment { self.layers.get(layer_id.0) } + /// Per-layer property-presence schema for this in-memory segment. Returns + /// an empty schema if the layer has no container in this segment. + pub fn layer_schema( + &self, + layer_id: LayerId, + ) -> raphtory_api::core::entities::properties::layer_schema::LayerPropSchema { + self.layers + .get(layer_id.0) + .map(|c| c.layer_schema().clone()) + .unwrap_or_default() + } + + /// Iterator over (layer_id, schema) for every layer present in this segment. + pub fn layer_schemas( + &self, + ) -> impl Iterator< + Item = ( + LayerId, + &raphtory_api::core::entities::properties::layer_schema::LayerPropSchema, + ), + > { + self.layers + .iter() + .enumerate() + .map(|(i, c)| (LayerId(i), c.layer_schema())) + } + pub fn degree(&self, n: LocalPOS, layer_id: LayerId, dir: Direction) -> usize { self.get_adj(n, layer_id).map_or(0, |adj| adj.degree(dir)) } diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs new file mode 100644 index 0000000000..9bcf766b4a --- /dev/null +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -0,0 +1,163 @@ +/// Per-segment / per-layer summary of which property ids have values in a +/// given (layer, segment) pair. +/// +/// `temporal_props[i] == true` means a value has been written for global +/// temporal-prop-id `i` in this segment for this layer. Same for `metadata` +/// with the global metadata-prop-id space. +/// +/// Bitsets are merged across segments and layers by `union_with`, giving the +/// per-layer property schema without scanning entities. +/// +/// The representation is intentionally simple — a `Vec` rather than a +/// packed bitset — because the schema is small (number of properties, not +/// number of rows) and needs to round-trip through disk via rkyv. We can swap +/// in a denser representation later if profiling justifies it. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct LayerPropSchema { + temporal_props: Vec, + metadata: Vec, +} + +impl LayerPropSchema { + pub fn new() -> Self { + Self::default() + } + + /// Build a schema directly from two presence vectors. Used when + /// reconstructing a schema from persisted on-disk state. + pub fn from_bools(temporal_props: Vec, metadata: Vec) -> Self { + Self { + temporal_props, + metadata, + } + } + + /// Mark a temporal property id as present. + #[inline] + pub fn insert_temporal(&mut self, prop_id: usize) { + if prop_id >= self.temporal_props.len() { + self.temporal_props.resize(prop_id + 1, false); + } + self.temporal_props[prop_id] = true; + } + + /// Mark a metadata (const) property id as present. + #[inline] + pub fn insert_metadata(&mut self, prop_id: usize) { + if prop_id >= self.metadata.len() { + self.metadata.resize(prop_id + 1, false); + } + self.metadata[prop_id] = true; + } + + #[inline] + pub fn contains_temporal(&self, prop_id: usize) -> bool { + self.temporal_props.get(prop_id).copied().unwrap_or(false) + } + + #[inline] + pub fn contains_metadata(&self, prop_id: usize) -> bool { + self.metadata.get(prop_id).copied().unwrap_or(false) + } + + pub fn temporal_prop_ids(&self) -> impl Iterator + '_ { + self.temporal_props + .iter() + .enumerate() + .filter_map(|(i, &b)| b.then_some(i)) + } + + pub fn metadata_prop_ids(&self) -> impl Iterator + '_ { + self.metadata + .iter() + .enumerate() + .filter_map(|(i, &b)| b.then_some(i)) + } + + pub fn temporal_bits(&self) -> &[bool] { + &self.temporal_props + } + + pub fn metadata_bits(&self) -> &[bool] { + &self.metadata + } + + /// Union another schema into this one. + pub fn union_with(&mut self, other: &LayerPropSchema) { + union_into(&mut self.temporal_props, &other.temporal_props); + union_into(&mut self.metadata, &other.metadata); + } + + pub fn is_empty(&self) -> bool { + !self.temporal_props.iter().any(|&b| b) && !self.metadata.iter().any(|&b| b) + } +} + +fn union_into(dst: &mut Vec, src: &[bool]) { + if src.len() > dst.len() { + dst.resize(src.len(), false); + } + for (d, s) in dst.iter_mut().zip(src.iter()) { + *d |= *s; + } +} + +impl FromIterator for LayerPropSchema { + fn from_iter>(iter: I) -> Self { + let mut acc = LayerPropSchema::new(); + for s in iter { + acc.union_with(&s); + } + acc + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn insert_and_query() { + let mut s = LayerPropSchema::new(); + s.insert_temporal(0); + s.insert_temporal(5); + s.insert_metadata(2); + + assert!(s.contains_temporal(0)); + assert!(s.contains_temporal(5)); + assert!(!s.contains_temporal(1)); + assert!(s.contains_metadata(2)); + assert!(!s.contains_metadata(0)); + + let t: Vec = s.temporal_prop_ids().collect(); + assert_eq!(t, vec![0, 5]); + let m: Vec = s.metadata_prop_ids().collect(); + assert_eq!(m, vec![2]); + } + + #[test] + fn union() { + let mut a = LayerPropSchema::new(); + a.insert_temporal(1); + a.insert_metadata(3); + + let mut b = LayerPropSchema::new(); + b.insert_temporal(10); + b.insert_metadata(3); + b.insert_metadata(7); + + a.union_with(&b); + assert_eq!(a.temporal_prop_ids().collect::>(), vec![1, 10]); + assert_eq!(a.metadata_prop_ids().collect::>(), vec![3, 7]); + } + + #[test] + fn from_bools_roundtrip() { + let mut s = LayerPropSchema::new(); + s.insert_temporal(2); + s.insert_metadata(0); + let rebuilt = + LayerPropSchema::from_bools(s.temporal_bits().to_vec(), s.metadata_bits().to_vec()); + assert_eq!(s, rebuilt); + } +} diff --git a/raphtory-api/src/core/entities/properties/mod.rs b/raphtory-api/src/core/entities/properties/mod.rs index f20788ef0f..ebba50fdc4 100644 --- a/raphtory-api/src/core/entities/properties/mod.rs +++ b/raphtory-api/src/core/entities/properties/mod.rs @@ -1,3 +1,4 @@ +pub mod layer_schema; pub mod meta; pub mod prop; pub mod tprop; From 934cd2b0df62b744913fefaa5b7a34e37aeaa095 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 2 Jun 2026 02:45:22 -0400 Subject: [PATCH 02/27] Changed graphql schema generation to use the bitsets (Vec for now) of properties present in each (segment, layer) to efficiently retrieve the lists of relevant properties. We union over all segments in the storage. We also filter out props/metadata that aren't present now instead of all globally registered properties. --- db4-storage/src/pages/node_store.rs | 4 ++ .../core/entities/properties/layer_schema.rs | 37 +++++++++++ .../src/model/schema/edge_schema.rs | 66 ++++++++++++++++--- .../src/model/schema/node_schema.rs | 10 ++- raphtory-storage/src/graph/edges/edges.rs | 23 ++++++- raphtory-storage/src/graph/nodes/nodes_ref.rs | 20 +++++- raphtory/src/db/api/properties/internal.rs | 50 +++++++++++++- .../graph/storage_ops/property_schema.rs | 16 ++++- raphtory/src/db/api/view/graph.rs | 1 - .../db/graph/views/property_redacted_graph.rs | 35 +++++++++- 10 files changed, 246 insertions(+), 16 deletions(-) diff --git a/db4-storage/src/pages/node_store.rs b/db4-storage/src/pages/node_store.rs index 89413ef277..e42287df1f 100644 --- a/db4-storage/src/pages/node_store.rs +++ b/db4-storage/src/pages/node_store.rs @@ -52,6 +52,10 @@ pub struct ReadLockedNodeStorage, EXT> { impl, EXT: PersistenceStrategy> ReadLockedNodeStorage { + pub fn storage(&self) -> &NodeStorageInner { + &self.storage + } + pub fn node_ref( &self, node: impl Into, diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs index 9bcf766b4a..8a3b06e734 100644 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -88,6 +88,19 @@ impl LayerPropSchema { union_into(&mut self.metadata, &other.metadata); } + /// Position-wise AND of the temporal-prop bits with the supplied + /// visibility mask. Bits at positions beyond `mask.len()` are cleared + /// (treated as "not visible"). + pub fn intersect_temporal_with(&mut self, mask: &[bool]) { + intersect_into(&mut self.temporal_props, mask); + } + + /// Position-wise AND of the metadata bits with the supplied visibility + /// mask. See [`Self::intersect_temporal_with`]. + pub fn intersect_metadata_with(&mut self, mask: &[bool]) { + intersect_into(&mut self.metadata, mask); + } + pub fn is_empty(&self) -> bool { !self.temporal_props.iter().any(|&b| b) && !self.metadata.iter().any(|&b| b) } @@ -102,6 +115,12 @@ fn union_into(dst: &mut Vec, src: &[bool]) { } } +fn intersect_into(dst: &mut Vec, mask: &[bool]) { + for (i, d) in dst.iter_mut().enumerate() { + *d &= mask.get(i).copied().unwrap_or(false); + } +} + impl FromIterator for LayerPropSchema { fn from_iter>(iter: I) -> Self { let mut acc = LayerPropSchema::new(); @@ -151,6 +170,24 @@ mod tests { assert_eq!(a.metadata_prop_ids().collect::>(), vec![3, 7]); } + #[test] + fn intersect() { + let mut s = LayerPropSchema::new(); + s.insert_temporal(0); + s.insert_temporal(2); + s.insert_temporal(5); + s.insert_metadata(1); + s.insert_metadata(3); + + // Mask shorter than schema — trailing bits clear. + s.intersect_temporal_with(&[true, false, true]); + assert_eq!(s.temporal_prop_ids().collect::>(), vec![0, 2]); + + // Mask covers schema — keep only marked bits. + s.intersect_metadata_with(&[false, false, false, true, true]); + assert_eq!(s.metadata_prop_ids().collect::>(), vec![3]); + } + #[test] fn from_bools_roundtrip() { let mut s = LayerPropSchema::new(); diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 463980db89..6470443564 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -7,10 +7,14 @@ use crate::{ use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use itertools::Itertools; use raphtory::{ - db::{api::view::StaticGraphViewOps, graph::edge::EdgeView}, + db::{ + api::{properties::internal::EdgePropertySchemaOps, view::StaticGraphViewOps}, + graph::edge::EdgeView, + }, prelude::*, }; -use raphtory_api::core::entities::properties::meta::PropMapper; +use raphtory_api::core::entities::properties::{layer_schema::LayerPropSchema, meta::PropMapper}; +use raphtory_storage::layer_ops::InternalLayerOps; use std::collections::HashSet; /// Describes edges between a specific pair of node types — the property and @@ -53,34 +57,54 @@ impl EdgeSchema { self.dst_type.clone() } - /// Returns the list of property schemas for edges connecting these types of nodes + /// Returns the list of property schemas for edges connecting these types of nodes. + /// + /// Edges are filtered by `(src_type, dst_type)` and their temporal + /// properties are aggregated into per-key value-variant sets (preserved + /// up to `ENUM_BOUNDARY` distinct values per key, see `merge_schemas`). + /// The resulting key set is intersected with the per-layer property + /// bitset so anything not actually present in this layer is dropped. async fn properties(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let schema: SchemaAggregate = cloned + let mapper = cloned.graph.edge_meta().temporal_prop_mapper(); + let layers = cloned.graph.layer_ids().clone(); + let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); + let aggregate: SchemaAggregate = cloned .edges() .map(collect_edge_property_schema) .reduce(merge_schemas) .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + aggregate_to_property_list(aggregate, mapper, &layer_schema, PropKind::Temporal) }) .await } - /// Returns the list of metadata schemas for edges connecting these types of nodes + /// Returns the list of metadata schemas for edges connecting these types of nodes. + /// Same shape as `properties` but over metadata fields rather than + /// temporal properties. async fn metadata(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let schema: SchemaAggregate = cloned + let mapper = cloned.graph.edge_meta().metadata_mapper(); + let layers = cloned.graph.layer_ids().clone(); + let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); + let aggregate: SchemaAggregate = cloned .edges() .map(collect_edge_metadata_schema) .reduce(merge_schemas) .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + aggregate_to_property_list(aggregate, mapper, &layer_schema, PropKind::Metadata) }) .await } } +#[derive(Copy, Clone)] +enum PropKind { + Temporal, + Metadata, +} + fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggregate { props .iter() @@ -114,3 +138,29 @@ fn collect_edge_metadata_schema<'graph, G: GraphViewOps<'graph>>( let mapper = edge.graph.edge_meta().metadata_mapper(); collect_schema(props, mapper) } + +/// Convert an aggregate into the final `PropertySchema` list, dropping any +/// keys that aren't in the per-layer bitset. The intersection is the bitset +/// adaptation: edges of this `(src_type, dst_type)` may have surfaced +/// properties whose ids the layer bitset disagrees with (e.g. through a +/// filtered or redacted view) — those are removed here. +fn aggregate_to_property_list( + aggregate: SchemaAggregate, + mapper: &PropMapper, + layer_schema: &LayerPropSchema, + kind: PropKind, +) -> Vec { + aggregate + .into_iter() + .filter(|((key, _dtype), _values)| { + mapper + .get_id(key) + .map(|id| match kind { + PropKind::Temporal => layer_schema.contains_temporal(id), + PropKind::Metadata => layer_schema.contains_metadata(id), + }) + .unwrap_or(false) + }) + .map(|prop| prop.into()) + .collect_vec() +} diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 6f8e5f61c3..1368ab6fe8 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -64,6 +64,11 @@ impl NodeSchema { .unwrap_or_else(|| DEFAULT_NODE_TYPE.to_string()) } fn properties_inner(&self) -> Vec { + // Per-layer presence schema, unioned across every layer — gives us the + // set of temporal prop ids that have actually been written, without + // scanning nodes. Intersected with the (redaction-aware) visibility + // set so that hidden ids are filtered out. + let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_temporal_prop_ids().collect(); let (keys, property_types): (Vec<_>, Vec<_>) = self @@ -72,7 +77,7 @@ impl NodeSchema { .temporal_prop_mapper() .locked() .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id)) + .filter(|(id, _, _)| visible.contains(id) && layer_schema.contains_temporal(*id)) .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); @@ -114,6 +119,7 @@ impl NodeSchema { } fn metadata_inner(&self) -> Vec { + let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_metadata_ids().collect(); let (keys, property_types): (Vec<_>, Vec<_>) = self @@ -122,7 +128,7 @@ impl NodeSchema { .metadata_mapper() .locked() .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id)) + .filter(|(id, _, _)| visible.contains(id) && layer_schema.contains_metadata(*id)) .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 1d96f8792f..7708e1f518 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -1,6 +1,9 @@ use super::{edge_entry::EdgeStorageEntry, unlocked::UnlockedEdges}; use either::Either; -use raphtory_api::core::entities::{properties::meta::STATIC_GRAPH_LAYER_ID, LayerIds, EID}; +use raphtory_api::core::entities::{ + properties::{layer_schema::LayerPropSchema, meta::STATIC_GRAPH_LAYER_ID}, + LayerIds, EID, +}; use raphtory_core::entities::edges::edge_ref::EdgeRef; use rayon::iter::ParallelIterator; use std::sync::Arc; @@ -145,4 +148,22 @@ impl<'a> EdgesStorageRef<'a> { EdgesStorageRef::Unlocked(storage) => storage.storage().num_edges(), } } + + /// Union of per-(segment, layer) edge property-presence schemas across the + /// supplied layers. Reads from the persisted `LayerStats` on disk and the + /// incrementally-tracked schema on the mem head — no row scanning. + pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + let inner = match self { + EdgesStorageRef::Mem(storage) => storage.storage(), + EdgesStorageRef::Unlocked(storage) => storage.storage(), + }; + let num_layers = inner.num_layers(); + let mut schema = LayerPropSchema::new(); + for (_, seg) in inner.segments().iter() { + for layer_id in layers.iter(num_layers) { + schema.union_with(&seg.layer_schema(layer_id)); + } + } + schema + } } diff --git a/raphtory-storage/src/graph/nodes/nodes_ref.rs b/raphtory-storage/src/graph/nodes/nodes_ref.rs index f170f8dafd..4d12bcafd1 100644 --- a/raphtory-storage/src/graph/nodes/nodes_ref.rs +++ b/raphtory-storage/src/graph/nodes/nodes_ref.rs @@ -1,6 +1,6 @@ use super::node_ref::NodeStorageRef; use crate::graph::variants::storage_variants3::StorageVariants3; -use raphtory_api::core::entities::VID; +use raphtory_api::core::entities::{properties::layer_schema::LayerPropSchema, LayerIds, VID}; use rayon::iter::ParallelIterator; use storage::{Extension, ReadLockedNodes}; @@ -45,6 +45,24 @@ impl<'a> NodesStorageEntry<'a> { for_all_variants!(self, nodes => nodes.iter()) } + /// Union of per-(segment, layer) node property-presence schemas across the + /// supplied layers. Reads from the persisted `LayerStats` on disk and the + /// incrementally-tracked schema on the mem head — no row scanning. + pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + let inner = match self { + NodesStorageEntry::Mem(nodes) => nodes.storage(), + NodesStorageEntry::Unlocked(nodes) => nodes.storage(), + }; + let num_layers = inner.num_layers(); + let mut schema = LayerPropSchema::new(); + for seg in inner.segments_iter() { + for layer_id in layers.iter(num_layers) { + schema.union_with(&seg.layer_schema(layer_id)); + } + } + schema + } + /// Returns a parallel iterator over nodes row groups /// the (usize) part is the row group not the segment pub fn row_groups_par_iter( diff --git a/raphtory/src/db/api/properties/internal.rs b/raphtory/src/db/api/properties/internal.rs index 9f61e55c86..ef07f3f434 100644 --- a/raphtory/src/db/api/properties/internal.rs +++ b/raphtory/src/db/api/properties/internal.rs @@ -1,7 +1,13 @@ use crate::db::api::view::BoxedLIter; use raphtory_api::{ core::{ - entities::properties::prop::{Prop, PropType}, + entities::{ + properties::{ + layer_schema::LayerPropSchema, + prop::{Prop, PropType}, + }, + LayerIds, + }, storage::{arc_str::ArcStr, timeindex::EventTime}, }, inherit::Base, @@ -20,6 +26,26 @@ pub trait NodePropertySchemaOps: Send + Sync { fn node_visible_metadata_id(&self, name: &str) -> Option; /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn node_visible_metadata_name(&self, id: usize) -> Option; + + /// Union of the per-(segment, layer) presence bitsets for all node + /// properties present in the supplied layers. + /// + /// Storage-backed implementations (see `GraphStorage`) read the persisted + /// `LayerStats` and union across segments — O(num props), no row + /// scanning. The default implementation falls back to the + /// `_visible_*_prop_ids` iterators (which return all globally registered + /// ids), preserving the historical behavior for view-only types that + /// don't reach storage. + fn node_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { + let mut schema = LayerPropSchema::new(); + for id in self.node_visible_temporal_prop_ids() { + schema.insert_temporal(id); + } + for id in self.node_visible_metadata_ids() { + schema.insert_metadata(id); + } + schema + } } /// Same as `NodePropertySchemaOps` but for edge properties. @@ -32,6 +58,20 @@ pub trait EdgePropertySchemaOps: Send + Sync { fn edge_visible_metadata_id(&self, name: &str) -> Option; /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn edge_visible_metadata_name(&self, id: usize) -> Option; + + /// Union of the per-(segment, layer) presence bitsets for all edge + /// properties present in the supplied layers. See + /// [`NodePropertySchemaOps::node_layer_prop_schema`]. + fn edge_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { + let mut schema = LayerPropSchema::new(); + for id in self.edge_visible_temporal_prop_ids() { + schema.insert_temporal(id); + } + for id in self.edge_visible_metadata_ids() { + schema.insert_metadata(id); + } + schema + } } /// Marker: delegate `NodePropertySchemaOps` through `Base`. @@ -67,6 +107,10 @@ where fn node_visible_metadata_name(&self, id: usize) -> Option { self.base().node_visible_metadata_name(id) } + #[inline] + fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + self.base().node_layer_prop_schema(layers) + } } impl EdgePropertySchemaOps for G @@ -97,6 +141,10 @@ where fn edge_visible_metadata_name(&self, id: usize) -> Option { self.base().edge_visible_metadata_name(id) } + #[inline] + fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + self.base().edge_layer_prop_schema(layers) + } } pub trait InternalTemporalPropertyViewOps { diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index 94f44793bc..ab4229963f 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -3,7 +3,13 @@ use crate::db::api::{ properties::internal::{EdgePropertySchemaOps, NodePropertySchemaOps}, view::BoxedLIter, }; -use raphtory_api::{core::storage::arc_str::ArcStr, iter::IntoDynBoxed}; +use raphtory_api::{ + core::{ + entities::{properties::layer_schema::LayerPropSchema, LayerIds}, + storage::arc_str::ArcStr, + }, + iter::IntoDynBoxed, +}; impl NodePropertySchemaOps for GraphStorage { fn node_visible_temporal_prop_ids(&self) -> BoxedLIter<'_, usize> { @@ -27,6 +33,10 @@ impl NodePropertySchemaOps for GraphStorage { fn node_visible_metadata_name(&self, id: usize) -> Option { Some(self.node_meta().metadata_mapper().get_name(id).clone()) } + + fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + self.nodes().layer_prop_schema(layers) + } } impl EdgePropertySchemaOps for GraphStorage { @@ -51,4 +61,8 @@ impl EdgePropertySchemaOps for GraphStorage { fn edge_visible_metadata_name(&self, id: usize) -> Option { Some(self.edge_meta().metadata_mapper().get_name(id).clone()) } + + fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + self.edges().layer_prop_schema(layers) + } } diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 666079e3cc..252533cee3 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -406,7 +406,6 @@ pub fn materialize_impl( let stream_capacity = 10; let (tx, rx) = crossbeam_channel::bounded::(stream_capacity); - // let mut scope_result = Ok(()); // Use std::thread::scope rather than rayon::scope so the producer runs on its own OS thread. // With rayon::scope on a single-thread pool, the main thread blocking on rx.recv() would starve the spawned producer. std::thread::scope(|scope| { diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index c06aa001d7..e0677b6ba9 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -13,7 +13,13 @@ use crate::db::api::{ }, }; use raphtory_api::{ - core::{entities::properties::prop::Prop, storage::arc_str::ArcStr}, + core::{ + entities::{ + properties::{layer_schema::LayerPropSchema, prop::Prop}, + LayerIds, + }, + storage::arc_str::ArcStr, + }, inherit::Base, }; use raphtory_storage::layer_ops::InheritLayerOps; @@ -247,6 +253,14 @@ impl NodePropertySchemaOps for PropertyRedactedGraph { } self.graph.node_visible_metadata_name(id) } + + fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + mask_schema( + self.graph.node_layer_prop_schema(layers), + &self.redaction.node_props_visible, + &self.redaction.node_meta_visible, + ) + } } impl EdgePropertySchemaOps for PropertyRedactedGraph { @@ -289,6 +303,25 @@ impl EdgePropertySchemaOps for PropertyRedactedGraph { } self.graph.edge_visible_metadata_name(id) } + + fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { + mask_schema( + self.graph.edge_layer_prop_schema(layers), + &self.redaction.edge_props_visible, + &self.redaction.edge_meta_visible, + ) + } +} + +/// Mask out redacted bits from a `LayerPropSchema` +fn mask_schema( + mut schema: LayerPropSchema, + temporal_visible: &[bool], + metadata_visible: &[bool], +) -> LayerPropSchema { + schema.intersect_temporal_with(temporal_visible); + schema.intersect_metadata_with(metadata_visible); + schema } // Graph-level property redaction: override InternalTemporalPropertiesOps and InternalMetadataOps From f20fc4f40c1a6b764313a369d0c94029e3d83d37 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 3 Jun 2026 05:15:17 -0400 Subject: [PATCH 03/27] Cleanup and add FIXME comments --- .../core/entities/properties/layer_schema.rs | 23 ++---- .../src/model/schema/edge_schema.rs | 82 +++++++------------ .../src/model/schema/node_schema.rs | 14 +++- raphtory-storage/src/graph/edges/edges.rs | 2 +- raphtory-storage/src/graph/nodes/nodes_ref.rs | 2 +- raphtory/src/db/api/properties/internal.rs | 16 +--- .../db/graph/views/property_redacted_graph.rs | 2 +- 7 files changed, 53 insertions(+), 88 deletions(-) diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs index 8a3b06e734..c827de08e8 100644 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -1,17 +1,9 @@ -/// Per-segment / per-layer summary of which property ids have values in a -/// given (layer, segment) pair. +/// Per-segment / per-layer summary of which property ids have values in a given (layer, segment) pair. +/// The Vec "bitsets" round-trip through disk via rkyv in LayerStatStore. /// /// `temporal_props[i] == true` means a value has been written for global /// temporal-prop-id `i` in this segment for this layer. Same for `metadata` /// with the global metadata-prop-id space. -/// -/// Bitsets are merged across segments and layers by `union_with`, giving the -/// per-layer property schema without scanning entities. -/// -/// The representation is intentionally simple — a `Vec` rather than a -/// packed bitset — because the schema is small (number of properties, not -/// number of rows) and needs to round-trip through disk via rkyv. We can swap -/// in a denser representation later if profiling justifies it. #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct LayerPropSchema { temporal_props: Vec, @@ -23,7 +15,7 @@ impl LayerPropSchema { Self::default() } - /// Build a schema directly from two presence vectors. Used when + /// Build a schema directly from two presence vectors. Useful when /// reconstructing a schema from persisted on-disk state. pub fn from_bools(temporal_props: Vec, metadata: Vec) -> Self { Self { @@ -88,15 +80,14 @@ impl LayerPropSchema { union_into(&mut self.metadata, &other.metadata); } - /// Position-wise AND of the temporal-prop bits with the supplied - /// visibility mask. Bits at positions beyond `mask.len()` are cleared - /// (treated as "not visible"). + /// Position-wise AND of the temporal-prop bits with the supplied visibility mask. + /// Bits at positions beyond `mask.len()` are treated as not visible. pub fn intersect_temporal_with(&mut self, mask: &[bool]) { intersect_into(&mut self.temporal_props, mask); } - /// Position-wise AND of the metadata bits with the supplied visibility - /// mask. See [`Self::intersect_temporal_with`]. + /// Position-wise AND of the metadata bits with the supplied visibility mask. + /// Bits at positions beyond `mask.len()` are treated as not visible. pub fn intersect_metadata_with(&mut self, mask: &[bool]) { intersect_into(&mut self.metadata, mask); } diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 6470443564..0b08208248 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -57,59 +57,57 @@ impl EdgeSchema { self.dst_type.clone() } - /// Returns the list of property schemas for edges connecting these types of nodes. - /// - /// Edges are filtered by `(src_type, dst_type)` and their temporal - /// properties are aggregated into per-key value-variant sets (preserved - /// up to `ENUM_BOUNDARY` distinct values per key, see `merge_schemas`). - /// The resulting key set is intersected with the per-layer property - /// bitset so anything not actually present in this layer is dropped. + /// Returns the list of property schemas for edges matching these `(src_node_type, dst_node_type)` async fn properties(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let mapper = cloned.graph.edge_meta().temporal_prop_mapper(); let layers = cloned.graph.layer_ids().clone(); let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); - let aggregate: SchemaAggregate = cloned + let schema: SchemaAggregate = cloned .edges() - .map(collect_edge_property_schema) + .map(|e| collect_edge_property_schema(e, &layer_schema, false)) .reduce(merge_schemas) .unwrap_or_default(); - aggregate_to_property_list(aggregate, mapper, &layer_schema, PropKind::Temporal) + schema.into_iter().map(|prop| prop.into()).collect_vec() }) .await } - /// Returns the list of metadata schemas for edges connecting these types of nodes. - /// Same shape as `properties` but over metadata fields rather than - /// temporal properties. + /// Returns the list of metadata schemas for edges matching these `(src_node_type, dst_node_type)` async fn metadata(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let mapper = cloned.graph.edge_meta().metadata_mapper(); let layers = cloned.graph.layer_ids().clone(); let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); - let aggregate: SchemaAggregate = cloned + let schema: SchemaAggregate = cloned .edges() - .map(collect_edge_metadata_schema) + .map(|e| collect_edge_metadata_schema(e, &layer_schema, true)) .reduce(merge_schemas) .unwrap_or_default(); - aggregate_to_property_list(aggregate, mapper, &layer_schema, PropKind::Metadata) + schema.into_iter().map(|prop| prop.into()).collect_vec() }) .await } } -#[derive(Copy, Clone)] -enum PropKind { - Temporal, - Metadata, -} - -fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggregate { +fn collect_schema( + props: P, + mapper: &PropMapper, + layer_schema: &LayerPropSchema, + is_metadata: bool, +) -> SchemaAggregate { props .iter() .zip(props.ids()) .filter_map(|((key, value), id)| { + // skip properties not in the layer schema + // FIXME: Is this even necessary? + // `edge.properties()` should only return properties present somewhere in the graph (by definition) + if (is_metadata && !layer_schema.contains_metadata(id)) + || (!is_metadata && !layer_schema.contains_temporal(id)) + { + return None; + } + let value = value?; let key_with_prop_type = ( key.to_string(), @@ -125,42 +123,20 @@ fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggr fn collect_edge_property_schema<'graph, G: GraphViewOps<'graph>>( edge: EdgeView, + layer_schema: &LayerPropSchema, + is_metadata: bool, ) -> SchemaAggregate { let props = edge.properties(); let mapper = edge.graph.edge_meta().temporal_prop_mapper(); - collect_schema(props, mapper) + collect_schema(props, mapper, layer_schema, is_metadata) } fn collect_edge_metadata_schema<'graph, G: GraphViewOps<'graph>>( edge: EdgeView, + layer_schema: &LayerPropSchema, + is_metadata: bool, ) -> SchemaAggregate { let props = edge.metadata(); let mapper = edge.graph.edge_meta().metadata_mapper(); - collect_schema(props, mapper) -} - -/// Convert an aggregate into the final `PropertySchema` list, dropping any -/// keys that aren't in the per-layer bitset. The intersection is the bitset -/// adaptation: edges of this `(src_type, dst_type)` may have surfaced -/// properties whose ids the layer bitset disagrees with (e.g. through a -/// filtered or redacted view) — those are removed here. -fn aggregate_to_property_list( - aggregate: SchemaAggregate, - mapper: &PropMapper, - layer_schema: &LayerPropSchema, - kind: PropKind, -) -> Vec { - aggregate - .into_iter() - .filter(|((key, _dtype), _values)| { - mapper - .get_id(key) - .map(|id| match kind { - PropKind::Temporal => layer_schema.contains_temporal(id), - PropKind::Metadata => layer_schema.contains_metadata(id), - }) - .unwrap_or(false) - }) - .map(|prop| prop.into()) - .collect_vec() + collect_schema(props, mapper, layer_schema, is_metadata) } diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 1368ab6fe8..ecb92cbd04 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -64,10 +64,11 @@ impl NodeSchema { .unwrap_or_else(|| DEFAULT_NODE_TYPE.to_string()) } fn properties_inner(&self) -> Vec { - // Per-layer presence schema, unioned across every layer — gives us the - // set of temporal prop ids that have actually been written, without - // scanning nodes. Intersected with the (redaction-aware) visibility - // set so that hidden ids are filtered out. + // Property presence schema across every layer. Shouldn't make a difference for normal `GraphStorage`. + // Masks redacted properties from `PropertyRedactedGraph`. + // FIXME: Shouldn't all ids in the PropMapper exist somewhere in the graph, thus be present in the LayerPropSchema? + // FIXME: For the PropertyRedactedGraph, isn't the same filtering already being applied in `node_visible_temporal_prop_ids`? + // FIXME: Do we even wanna do this? let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_temporal_prop_ids().collect(); @@ -119,6 +120,11 @@ impl NodeSchema { } fn metadata_inner(&self) -> Vec { + // Property presence schema across every layer. Shouldn't make a difference for normal `GraphStorage`. + // Masks redacted properties from `PropertyRedactedGraph`. + // FIXME: Shouldn't all ids in the PropMapper exist somewhere in the graph, thus be present in the LayerPropSchema? + // FIXME: For the PropertyRedactedGraph, isn't the same filtering already being applied in `node_visible_temporal_prop_ids`? + // FIXME: Do we even wanna do this? let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_metadata_ids().collect(); diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 7708e1f518..29febc3300 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -151,7 +151,7 @@ impl<'a> EdgesStorageRef<'a> { /// Union of per-(segment, layer) edge property-presence schemas across the /// supplied layers. Reads from the persisted `LayerStats` on disk and the - /// incrementally-tracked schema on the mem head — no row scanning. + /// incrementally-tracked schema on the mem head. pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { let inner = match self { EdgesStorageRef::Mem(storage) => storage.storage(), diff --git a/raphtory-storage/src/graph/nodes/nodes_ref.rs b/raphtory-storage/src/graph/nodes/nodes_ref.rs index 4d12bcafd1..d35d65714e 100644 --- a/raphtory-storage/src/graph/nodes/nodes_ref.rs +++ b/raphtory-storage/src/graph/nodes/nodes_ref.rs @@ -47,7 +47,7 @@ impl<'a> NodesStorageEntry<'a> { /// Union of per-(segment, layer) node property-presence schemas across the /// supplied layers. Reads from the persisted `LayerStats` on disk and the - /// incrementally-tracked schema on the mem head — no row scanning. + /// incrementally-tracked schema on the mem head. pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { let inner = match self { NodesStorageEntry::Mem(nodes) => nodes.storage(), diff --git a/raphtory/src/db/api/properties/internal.rs b/raphtory/src/db/api/properties/internal.rs index ef07f3f434..f1ef288dd0 100644 --- a/raphtory/src/db/api/properties/internal.rs +++ b/raphtory/src/db/api/properties/internal.rs @@ -27,15 +27,8 @@ pub trait NodePropertySchemaOps: Send + Sync { /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn node_visible_metadata_name(&self, id: usize) -> Option; - /// Union of the per-(segment, layer) presence bitsets for all node - /// properties present in the supplied layers. - /// - /// Storage-backed implementations (see `GraphStorage`) read the persisted - /// `LayerStats` and union across segments — O(num props), no row - /// scanning. The default implementation falls back to the - /// `_visible_*_prop_ids` iterators (which return all globally registered - /// ids), preserving the historical behavior for view-only types that - /// don't reach storage. + /// Union of the (segment, layer) property presence bitsets for all node properties present in the layers. + /// Override for behaviour other than "all node properties registered globally". fn node_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { let mut schema = LayerPropSchema::new(); for id in self.node_visible_temporal_prop_ids() { @@ -59,9 +52,8 @@ pub trait EdgePropertySchemaOps: Send + Sync { /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn edge_visible_metadata_name(&self, id: usize) -> Option; - /// Union of the per-(segment, layer) presence bitsets for all edge - /// properties present in the supplied layers. See - /// [`NodePropertySchemaOps::node_layer_prop_schema`]. + /// Union of the (segment, layer) property presence bitsets for all edge properties present in the layers. + /// Override for behaviour other than "all edge properties registered globally". fn edge_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { let mut schema = LayerPropSchema::new(); for id in self.edge_visible_temporal_prop_ids() { diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index e0677b6ba9..6e038e2e2c 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -313,7 +313,7 @@ impl EdgePropertySchemaOps for PropertyRedactedGraph { } } -/// Mask out redacted bits from a `LayerPropSchema` +/// Mask out redacted bits from a layer property schema fn mask_schema( mut schema: LayerPropSchema, temporal_visible: &[bool], From 3a36d30a20b809f5411efaf76cd1c8be9123d067 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 3 Jun 2026 05:32:45 -0400 Subject: [PATCH 04/27] More cleanup --- db4-storage/src/properties/mod.rs | 10 +++++----- db4-storage/src/segments/edge/segment.rs | 19 +++++-------------- db4-storage/src/segments/mod.rs | 4 +--- db4-storage/src/segments/node/segment.rs | 19 +++++-------------- .../core/entities/properties/layer_schema.rs | 6 +++--- 5 files changed, 19 insertions(+), 39 deletions(-) diff --git a/db4-storage/src/properties/mod.rs b/db4-storage/src/properties/mod.rs index 6c8647d66e..c23f9bd46a 100644 --- a/db4-storage/src/properties/mod.rs +++ b/db4-storage/src/properties/mod.rs @@ -24,6 +24,7 @@ use std::sync::Arc; pub mod props_meta_writer; +// Held by SegmentContainer, which corresponds to one layer in Mem(Edge/Node)Segment #[derive(Debug, Default)] pub struct Properties { c_properties: Vec, @@ -39,9 +40,8 @@ pub struct Properties { has_properties: bool, has_deletions: bool, pub additions_count: usize, - /// Per-segment record of which (global) property ids have ever been written - /// to this `Properties`. Maintained incrementally on every write so that - /// the layer schema can be derived without scanning entities. + /// Per-(segment, layer) record of which (global) property ids have ever been written + /// to this `Properties`. Maintained incrementally on every write. layer_schema: LayerPropSchema, } @@ -99,8 +99,7 @@ impl Properties { self.t_properties.num_columns() } - /// Returns the per-segment property-presence summary maintained - /// incrementally as `append_t_props` / `append_const_props` are called. + /// Returns the property-presence summary maintained incrementally by `append_t_props` / `append_const_props`. pub fn layer_schema(&self) -> &LayerPropSchema { &self.layer_schema } @@ -298,6 +297,7 @@ impl<'a> PropMutEntry<'a> { .. } = &mut *self.properties; + // Inspect fn (layer_schema.insert_temporal(id)) is called when this is pushed in t_prop_row below let tracked = props .into_iter() .inspect(|(prop_id, _)| layer_schema.insert_temporal(*prop_id)); diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index 91e8aad566..ce937fa437 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -16,6 +16,7 @@ use raphtory_api::core::{ entities::{ LayerId, VID, properties::{ + layer_schema::LayerPropSchema, meta::{Meta, STATIC_GRAPH_LAYER_ID}, prop::AsPropRef, }, @@ -134,26 +135,16 @@ impl MemEdgeSegment { } /// Per-layer property-presence schema for this in-memory segment. Returns - /// an empty schema if the layer has no container in this segment. - pub fn layer_schema( - &self, - layer_id: LayerId, - ) -> raphtory_api::core::entities::properties::layer_schema::LayerPropSchema { + /// an empty schema if the layer has no container in this segment (no matching SegmentContainer). + pub fn layer_schema(&self, layer_id: LayerId) -> LayerPropSchema { self.layers .get(layer_id.0) .map(|c| c.layer_schema().clone()) .unwrap_or_default() } - /// Iterator over (layer_id, schema) for every layer present in this segment. - pub fn layer_schemas( - &self, - ) -> impl Iterator< - Item = ( - LayerId, - &raphtory_api::core::entities::properties::layer_schema::LayerPropSchema, - ), - > { + /// Iterator over (layer_id, schema) for every layer present in this in-memory segment. + pub fn layer_schemas(&self) -> impl Iterator { self.layers .iter() .enumerate() diff --git a/db4-storage/src/segments/mod.rs b/db4-storage/src/segments/mod.rs index e0451f7375..23fcf4d13e 100644 --- a/db4-storage/src/segments/mod.rs +++ b/db4-storage/src/segments/mod.rs @@ -251,9 +251,7 @@ impl SegmentContainer { &self.properties } - /// Per-layer property-presence summary for this container (one container = - /// one layer of one segment). Cheap — just borrows the incrementally - /// maintained bitsets. + /// Per-layer property-presence summary for this container (which corresponds to a single layer) pub fn layer_schema(&self) -> &LayerPropSchema { self.properties.layer_schema() } diff --git a/db4-storage/src/segments/node/segment.rs b/db4-storage/src/segments/node/segment.rs index 90cfc78e8a..43bf75ca19 100644 --- a/db4-storage/src/segments/node/segment.rs +++ b/db4-storage/src/segments/node/segment.rs @@ -17,6 +17,7 @@ use raphtory_api::core::{ entities::{ EID, LayerId, VID, properties::{ + layer_schema::LayerPropSchema, meta::Meta, prop::{AsPropRef, Prop}, }, @@ -151,26 +152,16 @@ impl MemNodeSegment { } /// Per-layer property-presence schema for this in-memory segment. Returns - /// an empty schema if the layer has no container in this segment. - pub fn layer_schema( - &self, - layer_id: LayerId, - ) -> raphtory_api::core::entities::properties::layer_schema::LayerPropSchema { + /// an empty schema if the layer has no container in this segment (no matching SegmentContainer). + pub fn layer_schema(&self, layer_id: LayerId) -> LayerPropSchema { self.layers .get(layer_id.0) .map(|c| c.layer_schema().clone()) .unwrap_or_default() } - /// Iterator over (layer_id, schema) for every layer present in this segment. - pub fn layer_schemas( - &self, - ) -> impl Iterator< - Item = ( - LayerId, - &raphtory_api::core::entities::properties::layer_schema::LayerPropSchema, - ), - > { + /// Iterator over (layer_id, schema) for every layer present in this in-memory segment. + pub fn layer_schemas(&self) -> impl Iterator { self.layers .iter() .enumerate() diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs index c827de08e8..f901c22676 100644 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -1,5 +1,5 @@ /// Per-segment / per-layer summary of which property ids have values in a given (layer, segment) pair. -/// The Vec "bitsets" round-trip through disk via rkyv in LayerStatStore. +/// The Vec "bitsets" themselves round-trip via rkyv in LayerStatStore when disk storage is enabled. /// /// `temporal_props[i] == true` means a value has been written for global /// temporal-prop-id `i` in this segment for this layer. Same for `metadata` @@ -81,13 +81,13 @@ impl LayerPropSchema { } /// Position-wise AND of the temporal-prop bits with the supplied visibility mask. - /// Bits at positions beyond `mask.len()` are treated as not visible. + /// Careful: Bits at positions beyond `mask.len()` are treated as not visible. pub fn intersect_temporal_with(&mut self, mask: &[bool]) { intersect_into(&mut self.temporal_props, mask); } /// Position-wise AND of the metadata bits with the supplied visibility mask. - /// Bits at positions beyond `mask.len()` are treated as not visible. + /// Careful: Bits at positions beyond `mask.len()` are treated as not visible. pub fn intersect_metadata_with(&mut self, mask: &[bool]) { intersect_into(&mut self.metadata, mask); } From 197d0d67ab6fa2afd4a625f006e6d3cdaa1b5734 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Fri, 5 Jun 2026 04:29:22 -0400 Subject: [PATCH 05/27] Add comments on types for clarity --- db4-storage/src/properties/mod.rs | 2 +- raphtory-graphql/src/model/schema/edge_schema.rs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/db4-storage/src/properties/mod.rs b/db4-storage/src/properties/mod.rs index c23f9bd46a..4a5457f2ea 100644 --- a/db4-storage/src/properties/mod.rs +++ b/db4-storage/src/properties/mod.rs @@ -24,7 +24,7 @@ use std::sync::Arc; pub mod props_meta_writer; -// Held by SegmentContainer, which corresponds to one layer in Mem(Edge/Node)Segment +// Held by SegmentContainer, which corresponds to one layer in a Mem(Edge/Node)Segment #[derive(Debug, Default)] pub struct Properties { c_properties: Vec, diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 0b08208248..492c184479 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -102,6 +102,8 @@ fn collect_schema( // skip properties not in the layer schema // FIXME: Is this even necessary? // `edge.properties()` should only return properties present somewhere in the graph (by definition) + // For PropertyRedactedGraph, hidden properties are filtered out by InternalTemporalPropertiesOps + // and InternalMetadataOps. I think the layer_schema is useless here. if (is_metadata && !layer_schema.contains_metadata(id)) || (!is_metadata && !layer_schema.contains_temporal(id)) { From 39564e7955616f495b75decb45a559d64421a286 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 9 Jun 2026 04:32:38 -0400 Subject: [PATCH 06/27] Added property presence bitset (Vec) in PropMapper. We have one for properties and one for metadata in Meta. Wired it in so that layers where we know the property isn't present are skipped. --- db4-storage/src/pages/edge_page/writer.rs | 26 ++++++++ db4-storage/src/pages/node_page/writer.rs | 13 ++++ .../src/core/entities/properties/meta.rs | 63 +++++++++++++++++++ raphtory-storage/src/graph/edges/edges.rs | 41 +++++++++--- raphtory-storage/src/graph/nodes/nodes_ref.rs | 44 ++++++++++--- raphtory/src/db/api/properties/internal.rs | 30 ++++++++- .../graph/storage_ops/property_schema.rs | 18 +++++- .../internal/time_semantics/filtered_edge.rs | 13 +++- .../db/graph/views/property_redacted_graph.rs | 22 ++++++- 9 files changed, 249 insertions(+), 21 deletions(-) diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index 03c96cefce..ae4684abba 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -56,6 +56,14 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen layer_id: LayerId, ) -> LocalPOS { self.graph_stats.update_time(t.t()); + // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed in `insert_edge_internal` + // without needing to pass Meta to insert_edge_internal + let meta = self.writer.edge_meta().clone(); + let props = props.into_iter().inspect(move |(id, _)| { + meta.temporal_prop_mapper() + .mark_prop_in_layer(layer_id, *id); + }); if self .writer .insert_edge_internal(t, edge_pos, src, dst, layer_id, props) @@ -131,6 +139,15 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen } } + // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed + let meta = self.writer.edge_meta().clone(); + let t_meta = meta.as_ref(); + let t_props = t_props.into_iter().inspect(move |(id, _)| { + t_meta + .temporal_prop_mapper() + .mark_prop_in_layer(layer_id, *id); + }); if self .writer .insert_edge_internal(t, edge_pos, src, dst, layer_id, t_props) @@ -141,6 +158,9 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen self.graph_stats.update_time(t.t()); + let c_props = c_props.into_iter().inspect(move |(id, _)| { + meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); + }); self.writer .update_const_properties(edge_pos, src, dst, layer_id, c_props); } @@ -212,6 +232,12 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen if !existing_edge { self.increment_layer_num_edges(layer_id); } + // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed (in update_const_properties) + let meta = self.writer.edge_meta().clone(); + let props = props.into_iter().inspect(move |(id, _)| { + meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); + }); self.writer .update_const_properties(edge_pos, src, dst, layer_id, props); } diff --git a/db4-storage/src/pages/node_page/writer.rs b/db4-storage/src/pages/node_page/writer.rs index 4b07c8b634..cfc5692025 100644 --- a/db4-storage/src/pages/node_page/writer.rs +++ b/db4-storage/src/pages/node_page/writer.rs @@ -155,6 +155,13 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri props: impl IntoIterator, ) { self.l_counter.update_time(t.t()); + // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed + let meta = self.mut_segment.node_meta().clone(); + let props = props.into_iter().inspect(move |(id, _)| { + meta.temporal_prop_mapper() + .mark_prop_in_layer(layer_id, *id); + }); let (is_new_node, add) = self.mut_segment.add_props(t, pos, layer_id, props); self.mut_segment.increment_est_size(add); if is_new_node && !self.page.has_node(pos, layer_id) { @@ -178,6 +185,12 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri layer_id: LayerId, props: impl IntoIterator, ) { + // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed + let meta = self.mut_segment.node_meta().clone(); + let props = props.into_iter().inspect(move |(id, _)| { + meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); + }); let (is_new_node, add) = self.mut_segment.update_metadata(pos, layer_id, props); self.mut_segment.increment_est_size(add); if is_new_node && !self.page.has_node(pos, layer_id) { diff --git a/raphtory-api/src/core/entities/properties/meta.rs b/raphtory-api/src/core/entities/properties/meta.rs index aaa932c236..bfd4422461 100644 --- a/raphtory-api/src/core/entities/properties/meta.rs +++ b/raphtory-api/src/core/entities/properties/meta.rs @@ -232,6 +232,18 @@ impl Meta { self.temporal_prop_mapper.get_name(prop_id) } } + + /// O(1) check: has temporal-prop `prop_id` been observed in `layer_id`? + #[inline] + pub fn temporal_layer_has(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.temporal_prop_mapper.layer_has(layer_id, prop_id) + } + + /// O(1) check: has metadata-prop `prop_id` been observed in `layer_id`? + #[inline] + pub fn metadata_layer_has(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.metadata_mapper.layer_has(layer_id, prop_id) + } } /// Manages the mapping of property names to their IDs and types. @@ -245,6 +257,10 @@ pub struct PropMapper { /// Estimated size in bytes of a single row of properties maintained by this mapper. row_size: AtomicUsize, + + /// Per-layer property presence bitset; `layer_presence[layer_id][prop_id]` + /// is true iff this property has been observed in this layer + layer_presence: Arc>>>, } impl Deref for PropMapper { @@ -268,7 +284,31 @@ impl PropMapper { id_mapper: DictMapper::new_with_private_fields(fields), row_size: AtomicUsize::new(row_size), dtypes: Arc::new(RwLock::new(dtypes)), + layer_presence: Arc::new(RwLock::new(Vec::new())), + } + } + + /// O(1) check: has property `prop_id` ever been observed in `layer_id`? + /// `false` is authoritative; callers can safely skip column reads for + /// this (layer, prop). `true` means at least one entity in `layer_id` + /// has prop `prop_id`. + #[inline] + pub fn layer_has(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.layer_presence + .read_recursive() + .get(layer_id.0) + .and_then(|row| row.get(prop_id)) + .copied() + .unwrap_or(false) + } + + /// Mark `prop_id` as present in `layer_id`. only takes the write lock once per (layer, prop) + pub fn mark_prop_in_layer(&self, layer_id: LayerId, prop_id: usize) { + if self.layer_has(layer_id, prop_id) { + return; } + let mut guard = self.layer_presence.write(); + ensure_and_set(&mut guard, layer_id.0, prop_id); } pub fn d_types(&self) -> impl Deref> + '_ { @@ -277,10 +317,12 @@ impl PropMapper { pub fn deep_clone(&self) -> Self { let dtypes = self.dtypes.read_recursive().clone(); + let layer_presence = self.layer_presence.read_recursive().clone(); Self { id_mapper: self.id_mapper.deep_clone(), row_size: AtomicUsize::new(self.row_size.load(std::sync::atomic::Ordering::Relaxed)), dtypes: Arc::new(RwLock::new(dtypes)), + layer_presence: Arc::new(RwLock::new(layer_presence)), } } @@ -396,10 +438,23 @@ impl PropMapper { dict_mapper: self.id_mapper.write(), d_types: self.dtypes.write(), row_size: &self.row_size, + layer_presence: self.layer_presence.write(), } } } +#[inline] +fn ensure_and_set(presence: &mut Vec>, layer_idx: usize, prop_id: usize) { + if presence.len() <= layer_idx { + presence.resize_with(layer_idx + 1, Vec::new); + } + let row = &mut presence[layer_idx]; + if row.len() <= prop_id { + row.resize(prop_id + 1, false); + } + row[prop_id] = true; +} + /// Write-locked view of a [`PropMapper`]. pub struct WriteLockedPropMapper<'a> { /// Maps property names to their IDs. @@ -410,6 +465,9 @@ pub struct WriteLockedPropMapper<'a> { /// Estimated size in bytes of a single row of properties maintained by this mapper. row_size: &'a AtomicUsize, + + /// Per-layer property presence bitset + layer_presence: RwLockWriteGuard<'a, Vec>>, } impl<'a> WriteLockedPropMapper<'a> { @@ -503,6 +561,11 @@ impl<'a> WriteLockedPropMapper<'a> { ) -> Result>, PropError> { fast_proptype_check(self.dict_mapper.map(), &self.d_types, prop, dtype) } + + /// Mark `prop_id` as present in `layer_id` + pub fn mark_in_layer(&mut self, layer_id: LayerId, prop_id: usize) { + ensure_and_set(&mut *self.layer_presence, layer_id.0, prop_id); + } } pub struct LockedPropMapper<'a> { diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 29febc3300..0e3f2e482c 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -2,7 +2,7 @@ use super::{edge_entry::EdgeStorageEntry, unlocked::UnlockedEdges}; use either::Either; use raphtory_api::core::entities::{ properties::{layer_schema::LayerPropSchema, meta::STATIC_GRAPH_LAYER_ID}, - LayerIds, EID, + LayerId, LayerIds, EID, }; use raphtory_core::entities::edges::edge_ref::EdgeRef; use rayon::iter::ParallelIterator; @@ -149,21 +149,46 @@ impl<'a> EdgesStorageRef<'a> { } } - /// Union of per-(segment, layer) edge property-presence schemas across the - /// supplied layers. Reads from the persisted `LayerStats` on disk and the - /// incrementally-tracked schema on the mem head. + /// Union of edge property presence across the supplied layers pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { let inner = match self { EdgesStorageRef::Mem(storage) => storage.storage(), EdgesStorageRef::Unlocked(storage) => storage.storage(), }; + let meta = inner.edge_meta(); let num_layers = inner.num_layers(); let mut schema = LayerPropSchema::new(); - for (_, seg) in inner.segments().iter() { - for layer_id in layers.iter(num_layers) { - schema.union_with(&seg.layer_schema(layer_id)); - } + for layer_id in layers.iter(num_layers) { + meta.temporal_prop_mapper() + .for_each_in_layer(layer_id, |id| schema.insert_temporal(id)); + meta.metadata_mapper() + .for_each_in_layer(layer_id, |id| schema.insert_metadata(id)); } schema } + + /// O(1) check via the per-layer bitset cached on `Meta.temporal_prop_mapper`. + /// `false` is authoritative — callers can skip column reads for `(layer_id, prop_id)`. + pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + let inner = match self { + EdgesStorageRef::Mem(storage) => storage.storage(), + EdgesStorageRef::Unlocked(storage) => storage.storage(), + }; + inner + .edge_meta() + .temporal_prop_mapper() + .layer_has(layer_id, prop_id) + } + + /// O(1) check via the per-layer bitset cached on `Meta.metadata_mapper`. + pub fn layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + let inner = match self { + EdgesStorageRef::Mem(storage) => storage.storage(), + EdgesStorageRef::Unlocked(storage) => storage.storage(), + }; + inner + .edge_meta() + .metadata_mapper() + .layer_has(layer_id, prop_id) + } } diff --git a/raphtory-storage/src/graph/nodes/nodes_ref.rs b/raphtory-storage/src/graph/nodes/nodes_ref.rs index d35d65714e..bf0a790a65 100644 --- a/raphtory-storage/src/graph/nodes/nodes_ref.rs +++ b/raphtory-storage/src/graph/nodes/nodes_ref.rs @@ -1,6 +1,8 @@ use super::node_ref::NodeStorageRef; use crate::graph::variants::storage_variants3::StorageVariants3; -use raphtory_api::core::entities::{properties::layer_schema::LayerPropSchema, LayerIds, VID}; +use raphtory_api::core::entities::{ + properties::layer_schema::LayerPropSchema, LayerId, LayerIds, VID, +}; use rayon::iter::ParallelIterator; use storage::{Extension, ReadLockedNodes}; @@ -45,24 +47,50 @@ impl<'a> NodesStorageEntry<'a> { for_all_variants!(self, nodes => nodes.iter()) } - /// Union of per-(segment, layer) node property-presence schemas across the - /// supplied layers. Reads from the persisted `LayerStats` on disk and the - /// incrementally-tracked schema on the mem head. + /// Union of node property presence across the supplied layers. Reads + /// from the per-layer bitsets on `Meta.temporal_prop_mapper` and + /// `Meta.metadata_mapper`. pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { let inner = match self { NodesStorageEntry::Mem(nodes) => nodes.storage(), NodesStorageEntry::Unlocked(nodes) => nodes.storage(), }; + let meta = inner.prop_meta(); let num_layers = inner.num_layers(); let mut schema = LayerPropSchema::new(); - for seg in inner.segments_iter() { - for layer_id in layers.iter(num_layers) { - schema.union_with(&seg.layer_schema(layer_id)); - } + for layer_id in layers.iter(num_layers) { + meta.temporal_prop_mapper() + .for_each_in_layer(layer_id, |id| schema.insert_temporal(id)); + meta.metadata_mapper() + .for_each_in_layer(layer_id, |id| schema.insert_metadata(id)); } schema } + /// O(1) check + pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + let inner = match self { + NodesStorageEntry::Mem(nodes) => nodes.storage(), + NodesStorageEntry::Unlocked(nodes) => nodes.storage(), + }; + inner + .prop_meta() + .temporal_prop_mapper() + .layer_has(layer_id, prop_id) + } + + /// O(1) check + pub fn layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + let inner = match self { + NodesStorageEntry::Mem(nodes) => nodes.storage(), + NodesStorageEntry::Unlocked(nodes) => nodes.storage(), + }; + inner + .prop_meta() + .metadata_mapper() + .layer_has(layer_id, prop_id) + } + /// Returns a parallel iterator over nodes row groups /// the (usize) part is the row group not the segment pub fn row_groups_par_iter( diff --git a/raphtory/src/db/api/properties/internal.rs b/raphtory/src/db/api/properties/internal.rs index f1ef288dd0..8bfc4dff09 100644 --- a/raphtory/src/db/api/properties/internal.rs +++ b/raphtory/src/db/api/properties/internal.rs @@ -6,7 +6,7 @@ use raphtory_api::{ layer_schema::LayerPropSchema, prop::{Prop, PropType}, }, - LayerIds, + LayerId, LayerIds, }, storage::{arc_str::ArcStr, timeindex::EventTime}, }, @@ -39,6 +39,12 @@ pub trait NodePropertySchemaOps: Send + Sync { } schema } + + /// O(1) check: is temporal-prop `prop_id` present on any node in `layer_id`? + fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; + + /// O(1) check: is metadata-prop `prop_id` present on any node in `layer_id`? + fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool; } /// Same as `NodePropertySchemaOps` but for edge properties. @@ -64,6 +70,12 @@ pub trait EdgePropertySchemaOps: Send + Sync { } schema } + + /// O(1) check: is temporal-prop `prop_id` present on any edge in `layer_id`? + fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; + + /// O(1) check: is metadata-prop `prop_id` present on any edge in `layer_id`? + fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool; } /// Marker: delegate `NodePropertySchemaOps` through `Base`. @@ -103,6 +115,14 @@ where fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { self.base().node_layer_prop_schema(layers) } + #[inline] + fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.base().node_layer_has_temporal_prop(layer_id, prop_id) + } + #[inline] + fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.base().node_layer_has_metadata(layer_id, prop_id) + } } impl EdgePropertySchemaOps for G @@ -134,6 +154,14 @@ where self.base().edge_visible_metadata_name(id) } #[inline] + fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.base().edge_layer_has_temporal_prop(layer_id, prop_id) + } + #[inline] + fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.base().edge_layer_has_metadata(layer_id, prop_id) + } + #[inline] fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { self.base().edge_layer_prop_schema(layers) } diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index ab4229963f..67543fb80f 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -5,7 +5,7 @@ use crate::db::api::{ }; use raphtory_api::{ core::{ - entities::{properties::layer_schema::LayerPropSchema, LayerIds}, + entities::{properties::layer_schema::LayerPropSchema, LayerId, LayerIds}, storage::arc_str::ArcStr, }, iter::IntoDynBoxed, @@ -37,6 +37,14 @@ impl NodePropertySchemaOps for GraphStorage { fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { self.nodes().layer_prop_schema(layers) } + + fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.nodes().layer_has_temporal_prop(layer_id, prop_id) + } + + fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.nodes().layer_has_metadata(layer_id, prop_id) + } } impl EdgePropertySchemaOps for GraphStorage { @@ -65,4 +73,12 @@ impl EdgePropertySchemaOps for GraphStorage { fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { self.edges().layer_prop_schema(layers) } + + fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.edges().layer_has_temporal_prop(layer_id, prop_id) + } + + fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + self.edges().layer_has_metadata(layer_id, prop_id) + } } diff --git a/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs b/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs index a97d4e4fa5..6dc8f98dac 100644 --- a/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs +++ b/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs @@ -1,5 +1,8 @@ use crate::{ - db::api::view::internal::{FilterOps, FilterState, FilterVariants, GraphView}, + db::api::{ + properties::internal::EdgePropertySchemaOps, + view::internal::{FilterOps, FilterState, FilterVariants, GraphView}, + }, prelude::{GraphViewOps, LayerOps}, }; use either::Either; @@ -421,7 +424,10 @@ impl<'a> FilteredEdgeStorageOps<'a> for EdgeEntryRef<'a> { view: G, layer_ids: &'a LayerIds, ) -> impl Iterator)> + 'a { + let prune_view = view.clone(); self.filtered_layer_ids_iter(view.clone(), layer_ids) + // skip layers we know don't have any edges with the property + .filter(move |&layer_id| prune_view.edge_layer_has_temporal_prop(layer_id, prop_id)) .map(move |layer_id| { ( layer_id, @@ -439,7 +445,10 @@ impl<'a> FilteredEdgeStorageOps<'a> for EdgeEntryRef<'a> { let layer_ids = view.layer_ids(); let mut values = self .metadata_iter(layer_ids, prop_id) - .filter(|(layer, _)| layer_filter(*layer)); + // skip layers we know don't have any edge with the property + .filter(|(layer, _)| { + view.edge_layer_has_metadata(*layer, prop_id) && layer_filter(*layer) + }); if view.num_layers() > 1 { let mut values = values.peekable(); if values.peek().is_some() { diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index 6e038e2e2c..abe33c4d8c 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -16,7 +16,7 @@ use raphtory_api::{ core::{ entities::{ properties::{layer_schema::LayerPropSchema, prop::Prop}, - LayerIds, + LayerId, LayerIds, }, storage::arc_str::ArcStr, }, @@ -254,6 +254,16 @@ impl NodePropertySchemaOps for PropertyRedactedGraph { self.graph.node_visible_metadata_name(id) } + fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + is_visible(&self.redaction.node_props_visible, prop_id) + && self.graph.node_layer_has_temporal_prop(layer_id, prop_id) + } + + fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + is_visible(&self.redaction.node_meta_visible, prop_id) + && self.graph.node_layer_has_metadata(layer_id, prop_id) + } + fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { mask_schema( self.graph.node_layer_prop_schema(layers), @@ -304,6 +314,16 @@ impl EdgePropertySchemaOps for PropertyRedactedGraph { self.graph.edge_visible_metadata_name(id) } + fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { + is_visible(&self.redaction.edge_props_visible, prop_id) + && self.graph.edge_layer_has_temporal_prop(layer_id, prop_id) + } + + fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { + is_visible(&self.redaction.edge_meta_visible, prop_id) + && self.graph.edge_layer_has_metadata(layer_id, prop_id) + } + fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { mask_schema( self.graph.edge_layer_prop_schema(layers), From 5ea955fb5490ea52231386efa86e33a0bdc37a9f Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 9 Jun 2026 06:21:03 -0400 Subject: [PATCH 07/27] Added optimizations to skip layers when we know the property isn't found in this layer. Added these for the node storage and edge storage. Skip layers in tprop_iter_layers, which is used in time semantics. --- db4-storage/src/api/edges.rs | 3 + db4-storage/src/segments/edge/entry.rs | 13 ++- .../core/entities/properties/layer_schema.rs | 10 +++ .../src/core/entities/properties/meta.rs | 23 +++-- .../src/graph/edges/edge_storage_ops.rs | 24 +++++ raphtory-storage/src/graph/edges/edges.rs | 4 +- raphtory-storage/src/graph/mod.rs | 33 +++++++ .../src/graph/nodes/node_storage_ops.rs | 89 ++++++++++++------- raphtory-storage/src/graph/nodes/nodes_ref.rs | 8 +- 9 files changed, 160 insertions(+), 47 deletions(-) diff --git a/db4-storage/src/api/edges.rs b/db4-storage/src/api/edges.rs index 12b93af1ec..56bd112270 100644 --- a/db4-storage/src/api/edges.rs +++ b/db4-storage/src/api/edges.rs @@ -193,4 +193,7 @@ pub trait EdgeRefOps<'a>: Copy + Clone + Send + Sync { fn edge_id(&self) -> EID; fn edge_ref(self, dir: Dir) -> Option; + + /// Graph-wide edge `Meta` + fn edge_meta(&self) -> &Arc; } diff --git a/db4-storage/src/segments/edge/entry.rs b/db4-storage/src/segments/edge/entry.rs index ebce52a74d..3db2538fdb 100644 --- a/db4-storage/src/segments/edge/entry.rs +++ b/db4-storage/src/segments/edge/entry.rs @@ -5,7 +5,11 @@ use crate::{ generic_t_props::WithTProps, segments::{additions::MemAdditions, edge::segment::MemEdgeSegment}, }; -use raphtory_api::core::entities::{LayerId, edges::edge_ref::Dir, properties::prop::Prop}; +use raphtory_api::core::entities::{ + LayerId, + edges::edge_ref::Dir, + properties::{meta::Meta, prop::Prop}, +}; use raphtory_core::{ entities::{ EID, Multiple, VID, @@ -14,7 +18,7 @@ use raphtory_core::{ }, storage::timeindex::{EventTime, TimeIndexOps}, }; -use std::marker::PhantomData; +use std::{marker::PhantomData, sync::Arc}; #[derive(Debug)] pub struct MemEdgeEntry<'a, MES> { @@ -227,4 +231,9 @@ impl<'a> EdgeRefOps<'a> for MemEdgeRef<'a> { fn internal_num_layers(self) -> usize { self.es.as_ref().len() } + + #[inline] + fn edge_meta(&self) -> &Arc { + self.es.edge_meta() + } } diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs index f901c22676..9faa630c72 100644 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -80,6 +80,16 @@ impl LayerPropSchema { union_into(&mut self.metadata, &other.metadata); } + /// Union temporal property presence bits into this schema + pub fn union_temporal_with(&mut self, bits: &[bool]) { + union_into(&mut self.temporal_props, bits); + } + + /// Union metadata presence bits into this schema + pub fn union_metadata_with(&mut self, bits: &[bool]) { + union_into(&mut self.metadata, bits); + } + /// Position-wise AND of the temporal-prop bits with the supplied visibility mask. /// Careful: Bits at positions beyond `mask.len()` are treated as not visible. pub fn intersect_temporal_with(&mut self, mask: &[bool]) { diff --git a/raphtory-api/src/core/entities/properties/meta.rs b/raphtory-api/src/core/entities/properties/meta.rs index bfd4422461..c2e299d9b0 100644 --- a/raphtory-api/src/core/entities/properties/meta.rs +++ b/raphtory-api/src/core/entities/properties/meta.rs @@ -260,7 +260,7 @@ pub struct PropMapper { /// Per-layer property presence bitset; `layer_presence[layer_id][prop_id]` /// is true iff this property has been observed in this layer - layer_presence: Arc>>>, + layer_prop_presence: Arc>>>, } impl Deref for PropMapper { @@ -284,7 +284,7 @@ impl PropMapper { id_mapper: DictMapper::new_with_private_fields(fields), row_size: AtomicUsize::new(row_size), dtypes: Arc::new(RwLock::new(dtypes)), - layer_presence: Arc::new(RwLock::new(Vec::new())), + layer_prop_presence: Arc::new(RwLock::new(Vec::new())), } } @@ -294,7 +294,7 @@ impl PropMapper { /// has prop `prop_id`. #[inline] pub fn layer_has(&self, layer_id: LayerId, prop_id: usize) -> bool { - self.layer_presence + self.layer_prop_presence .read_recursive() .get(layer_id.0) .and_then(|row| row.get(prop_id)) @@ -307,22 +307,31 @@ impl PropMapper { if self.layer_has(layer_id, prop_id) { return; } - let mut guard = self.layer_presence.write(); + let mut guard = self.layer_prop_presence.write(); ensure_and_set(&mut guard, layer_id.0, prop_id); } + /// Run `f` against the raw `&[bool]` of `layer_id`'s prop presence row, borrowing under a single read lock. + /// `f` can't acquire a write lock on layer_prop_presence or else it will deadlock. + #[inline] + pub fn with_layer_prop_bits(&self, layer_id: LayerId, f: impl FnOnce(&[bool]) -> R) -> R { + let guard = self.layer_prop_presence.read_recursive(); + let bits: &[bool] = guard.get(layer_id.0).map(|v| v.as_slice()).unwrap_or(&[]); + f(bits) + } + pub fn d_types(&self) -> impl Deref> + '_ { self.dtypes.read_recursive() } pub fn deep_clone(&self) -> Self { let dtypes = self.dtypes.read_recursive().clone(); - let layer_presence = self.layer_presence.read_recursive().clone(); + let layer_presence = self.layer_prop_presence.read_recursive().clone(); Self { id_mapper: self.id_mapper.deep_clone(), row_size: AtomicUsize::new(self.row_size.load(std::sync::atomic::Ordering::Relaxed)), dtypes: Arc::new(RwLock::new(dtypes)), - layer_presence: Arc::new(RwLock::new(layer_presence)), + layer_prop_presence: Arc::new(RwLock::new(layer_presence)), } } @@ -438,7 +447,7 @@ impl PropMapper { dict_mapper: self.id_mapper.write(), d_types: self.dtypes.write(), row_size: &self.row_size, - layer_presence: self.layer_presence.write(), + layer_presence: self.layer_prop_presence.write(), } } } diff --git a/raphtory-storage/src/graph/edges/edge_storage_ops.rs b/raphtory-storage/src/graph/edges/edge_storage_ops.rs index 6c338ddb40..8a1c45a3ed 100644 --- a/raphtory-storage/src/graph/edges/edge_storage_ops.rs +++ b/raphtory-storage/src/graph/edges/edge_storage_ops.rs @@ -237,4 +237,28 @@ impl<'a> EdgeStorageOps<'a> for storage::EdgeEntryRef<'a> { fn metadata_layer(self, layer_id: LayerId, prop_id: usize) -> Option { EdgeRefOps::c_prop(self, layer_id, prop_id) } + + /// Layer-skip override: drop layers where we know the property isn't present + fn temporal_prop_iter( + self, + layer_ids: &'a LayerIds, + prop_id: usize, + ) -> impl Iterator)> + 'a { + let meta = EdgeRefOps::edge_meta(&self).clone(); + self.layer_ids_iter(layer_ids) + .filter(move |&layer_id| meta.temporal_layer_has(layer_id, prop_id)) + .map(move |id| (id, EdgeStorageOps::temporal_prop_layer(self, id, prop_id))) + } + + /// Layer-skip override: same shape as `temporal_prop_iter` but against the metadata mapper + fn metadata_iter( + self, + layer_ids: &'a LayerIds, + prop_id: usize, + ) -> impl Iterator + 'a { + let meta = EdgeRefOps::edge_meta(&self).clone(); + self.layer_ids_iter(layer_ids) + .filter(move |&layer_id| meta.metadata_layer_has(layer_id, prop_id)) + .filter_map(move |id| Some((id, EdgeStorageOps::metadata_layer(self, id, prop_id)?))) + } } diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 0e3f2e482c..daa4e05fc1 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -160,9 +160,9 @@ impl<'a> EdgesStorageRef<'a> { let mut schema = LayerPropSchema::new(); for layer_id in layers.iter(num_layers) { meta.temporal_prop_mapper() - .for_each_in_layer(layer_id, |id| schema.insert_temporal(id)); + .with_layer_prop_bits(layer_id, |bits| schema.union_temporal_with(bits)); meta.metadata_mapper() - .for_each_in_layer(layer_id, |id| schema.insert_metadata(id)); + .with_layer_prop_bits(layer_id, |bits| schema.union_metadata_with(bits)); } schema } diff --git a/raphtory-storage/src/graph/mod.rs b/raphtory-storage/src/graph/mod.rs index 13af057137..2c8788edff 100644 --- a/raphtory-storage/src/graph/mod.rs +++ b/raphtory-storage/src/graph/mod.rs @@ -3,3 +3,36 @@ pub mod graph; pub mod locked; pub mod nodes; pub mod variants; + +use raphtory_api::core::entities::{properties::meta::STATIC_GRAPH_LAYER_ID, LayerId, LayerIds}; +use raphtory_core::entities::LayerVariants; +use storage::utils::Iter3; + +/// Build an iterator over all layer ids we should visit when reading +/// node-style temporal data: `STATIC_GRAPH_LAYER_ID` is always included +/// (unlayered entities must be visible in every view), plus the layers +/// requested in `layer_ids`. +pub fn layer_ids_with_static( + num_layers: usize, + layer_ids: &LayerIds, +) -> impl Iterator + Send + Sync + 'static { + match layer_ids { + LayerIds::None => LayerVariants::None(std::iter::once(STATIC_GRAPH_LAYER_ID)), + LayerIds::All => LayerVariants::All((0..num_layers).map(LayerId)), + LayerIds::One(id) => { + if *id == STATIC_GRAPH_LAYER_ID { + LayerVariants::One(std::iter::once(*id)) + } else { + LayerVariants::Multiple(Iter3::I([STATIC_GRAPH_LAYER_ID, *id].into_iter())) + } + } + LayerIds::Multiple(ids) => { + if ids.contains(STATIC_GRAPH_LAYER_ID) { + LayerVariants::Multiple(Iter3::J(ids.clone().into_iter())) + } else { + let v = std::iter::once(STATIC_GRAPH_LAYER_ID).chain(ids.clone().into_iter()); + LayerVariants::Multiple(Iter3::K(v)) + } + } + } +} diff --git a/raphtory-storage/src/graph/nodes/node_storage_ops.rs b/raphtory-storage/src/graph/nodes/node_storage_ops.rs index ccb7ddee3e..5233ee21e0 100644 --- a/raphtory-storage/src/graph/nodes/node_storage_ops.rs +++ b/raphtory-storage/src/graph/nodes/node_storage_ops.rs @@ -1,15 +1,12 @@ +use crate::graph::layer_ids_with_static; use raphtory_api::core::{ - entities::{ - edges::edge_ref::EdgeRef, - properties::{meta::STATIC_GRAPH_LAYER_ID, prop::Prop}, - GidRef, LayerId, LayerIds, VID, - }, + entities::{edges::edge_ref::EdgeRef, properties::prop::Prop, GidRef, LayerId, LayerIds, VID}, storage::timeindex::TimeIndexOps, Direction, }; use raphtory_core::{entities::LayerVariants, storage::timeindex::EventTime}; use std::{borrow::Cow, ops::Range, sync::Arc}; -use storage::{api::nodes::NodeRefOps, gen_ts::LayerIter, utils::Iter3, NodeEntryRef}; +use storage::{api::nodes::NodeRefOps, gen_ts::LayerIter, NodeEntryRef}; pub trait NodeStorageOps<'a>: Copy + Sized + Send + Sync + 'a { fn degree(self, layers: &LayerIds, dir: Direction) -> usize; @@ -58,7 +55,7 @@ pub trait NodeStorageOps<'a>: Copy + Sized + Send + Sync + 'a { prop_id: usize, ) -> impl Iterator)> + 'a { self.layer_ids_iter(layer_ids) - .map(move |id| (id, self.temporal_prop_layer(id, prop_id))) + .map(move |id| (id, NodeStorageOps::temporal_prop_layer(self, id, prop_id))) } fn tprop(self, prop_id: usize) -> storage::NodeTProps<'a>; @@ -68,33 +65,14 @@ pub trait NodeStorageOps<'a>: Copy + Sized + Send + Sync + 'a { /// Iterate over `NodeTProps` for each layer specified by `layer_ids`, always /// including `STATIC_GRAPH_LAYER_ID` (the layer for nodes added without an - /// explicit layer name). This mirrors the behaviour of `layer_ids_with_static` - /// used for node additions: unlayered nodes must be visible in every view. + /// explicit layer name). Unlayered nodes must be visible in every view. fn tprop_iter_layers( self, layer_ids: &LayerIds, prop_id: usize, ) -> impl Iterator> + Send + Sync + 'a { - let layers = match layer_ids { - LayerIds::None => LayerVariants::None(std::iter::once(STATIC_GRAPH_LAYER_ID)), - LayerIds::All => LayerVariants::All((0..self.num_layers()).map(LayerId)), - LayerIds::One(id) => { - if *id == STATIC_GRAPH_LAYER_ID { - LayerVariants::One(std::iter::once(*id)) - } else { - LayerVariants::Multiple(Iter3::I([STATIC_GRAPH_LAYER_ID, *id].into_iter())) - } - } - LayerIds::Multiple(ids) => { - if ids.contains(STATIC_GRAPH_LAYER_ID) { - LayerVariants::Multiple(Iter3::J(ids.clone().into_iter())) - } else { - let v = std::iter::once(STATIC_GRAPH_LAYER_ID).chain(ids.clone().into_iter()); - LayerVariants::Multiple(Iter3::K(v)) - } - } - }; - layers.map(move |id| self.temporal_prop_layer(id, prop_id)) + layer_ids_with_static(self.num_layers(), layer_ids) + .map(move |id| self.temporal_prop_layer(id, prop_id)) } fn constant_prop_layer(self, layer_id: LayerId, prop_id: usize) -> Option; @@ -104,8 +82,9 @@ pub trait NodeStorageOps<'a>: Copy + Sized + Send + Sync + 'a { layer_ids: &'a LayerIds, prop_id: usize, ) -> impl Iterator + 'a { - self.layer_ids_iter(layer_ids) - .filter_map(move |id| Some((id, self.constant_prop_layer(id, prop_id)?))) + self.layer_ids_iter(layer_ids).filter_map(move |id| { + Some((id, NodeStorageOps::constant_prop_layer(self, id, prop_id)?)) + }) } fn temp_prop_rows_range( @@ -205,6 +184,54 @@ impl<'a> NodeStorageOps<'a> for NodeEntryRef<'a> { NodeRefOps::c_prop(self, layer_id, prop_id) } + /// Layer-skip override: drop layers whose per-layer property presence bitset in + /// Meta says `prop_id` has never been written + fn temporal_prop_iter( + self, + layer_ids: &'a LayerIds, + prop_id: usize, + ) -> impl Iterator)> + 'a { + let meta = NodeRefOps::node_meta(&self).clone(); + self.layer_ids_iter(layer_ids) + .filter(move |&layer_id| meta.temporal_layer_has(layer_id, prop_id)) + .map(move |layer_id| { + ( + layer_id, + NodeStorageOps::temporal_prop_layer(self, layer_id, prop_id), + ) + }) + } + + /// Layer-skip override: same shape as `temporal_prop_iter` but for the metadata mapper. + fn constant_prop_iter( + self, + layer_ids: &'a LayerIds, + prop_id: usize, + ) -> impl Iterator + 'a { + let meta = NodeRefOps::node_meta(&self).clone(); + self.layer_ids_iter(layer_ids) + .filter(move |&layer_id| meta.metadata_layer_has(layer_id, prop_id)) + .filter_map(move |layer_id| { + Some(( + layer_id, + NodeStorageOps::constant_prop_layer(self, layer_id, prop_id)?, + )) + }) + } + + /// Layer-skip override: drops layers that the per-layer bitset says don't carry `prop_id`. + /// All node temporal-prop reads in event/persistent semantics flow through here. + fn tprop_iter_layers( + self, + layer_ids: &LayerIds, + prop_id: usize, + ) -> impl Iterator> + Send + Sync + 'a { + let meta = NodeRefOps::node_meta(&self).clone(); + layer_ids_with_static(self.num_layers(), layer_ids) + .filter(move |&id| meta.temporal_layer_has(id, prop_id)) + .map(move |id| NodeStorageOps::temporal_prop_layer(self, id, prop_id)) + } + fn temp_prop_rows_range( self, w: Option>, diff --git a/raphtory-storage/src/graph/nodes/nodes_ref.rs b/raphtory-storage/src/graph/nodes/nodes_ref.rs index bf0a790a65..94a1cf6a34 100644 --- a/raphtory-storage/src/graph/nodes/nodes_ref.rs +++ b/raphtory-storage/src/graph/nodes/nodes_ref.rs @@ -47,9 +47,7 @@ impl<'a> NodesStorageEntry<'a> { for_all_variants!(self, nodes => nodes.iter()) } - /// Union of node property presence across the supplied layers. Reads - /// from the per-layer bitsets on `Meta.temporal_prop_mapper` and - /// `Meta.metadata_mapper`. + /// Union of node property presence across the supplied layers. pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { let inner = match self { NodesStorageEntry::Mem(nodes) => nodes.storage(), @@ -60,9 +58,9 @@ impl<'a> NodesStorageEntry<'a> { let mut schema = LayerPropSchema::new(); for layer_id in layers.iter(num_layers) { meta.temporal_prop_mapper() - .for_each_in_layer(layer_id, |id| schema.insert_temporal(id)); + .with_layer_prop_bits(layer_id, |bits| schema.union_temporal_with(bits)); meta.metadata_mapper() - .for_each_in_layer(layer_id, |id| schema.insert_metadata(id)); + .with_layer_prop_bits(layer_id, |bits| schema.union_metadata_with(bits)); } schema } From b00185a34ad67dde5fc5d7130c13fbe2ed235eaf Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 10 Jun 2026 04:28:10 -0400 Subject: [PATCH 08/27] Get rid of all bitsets Vec added on LayerStatStore and Properties. These were used to track property presence per-(segment, layer) on disk and in-memory. Now, we only keep track of property presence per layer. We skip layers where we know the property was added, but that's it. --- db4-storage/src/properties/mod.rs | 27 +------ db4-storage/src/segments/edge/segment.rs | 18 ----- db4-storage/src/segments/mod.rs | 6 -- db4-storage/src/segments/node/segment.rs | 18 ----- .../core/entities/properties/layer_schema.rs | 78 +++++-------------- 5 files changed, 22 insertions(+), 125 deletions(-) diff --git a/db4-storage/src/properties/mod.rs b/db4-storage/src/properties/mod.rs index 5f143ad1a8..1adb9755a4 100644 --- a/db4-storage/src/properties/mod.rs +++ b/db4-storage/src/properties/mod.rs @@ -6,7 +6,6 @@ use arrow_array::{ use arrow_schema::DECIMAL128_MAX_PRECISION; use bigdecimal::ToPrimitive; use raphtory_api::core::entities::properties::{ - layer_schema::LayerPropSchema, meta::PropMapper, prop::{ AsPropRef, Prop, PropRef, PropType, SerdeArrowList, SerdeArrowMap, @@ -40,9 +39,6 @@ pub struct Properties { has_properties: bool, has_deletions: bool, pub additions_count: usize, - /// Per-(segment, layer) record of which (global) property ids have ever been written - /// to this `Properties`. Maintained incrementally on every write. - layer_schema: LayerPropSchema, } pub(crate) struct PropMutEntry<'a> { @@ -99,11 +95,6 @@ impl Properties { self.t_properties.num_columns() } - /// Returns the property-presence summary maintained incrementally by `append_t_props` / `append_const_props`. - pub fn layer_schema(&self) -> &LayerPropSchema { - &self.layer_schema - } - pub fn num_c_columns(&self) -> usize { self.c_properties.len() } @@ -312,19 +303,10 @@ impl<'a> PropMutEntry<'a> { t: EventTime, props: impl IntoIterator, ) { - let Properties { - t_properties, - layer_schema, - .. - } = &mut *self.properties; - - // Inspect fn (layer_schema.insert_temporal(id)) is called when this is pushed in t_prop_row below - let tracked = props - .into_iter() - .inspect(|(prop_id, _)| layer_schema.insert_temporal(*prop_id)); - - let t_prop_row = if let Some(t_prop_row) = t_properties - .push(tracked) + let t_prop_row = if let Some(t_prop_row) = self + .properties + .t_properties + .push(props) .expect("Internal error: properties should be validated at this point") { t_prop_row @@ -393,7 +375,6 @@ impl<'a> PropMutEntry<'a> { let const_props = &mut self.properties.c_properties[prop_id]; // property types should have been validated before! const_props.upsert(self.row, prop.as_prop_ref()).unwrap(); - self.properties.layer_schema.insert_metadata(prop_id); } } } diff --git a/db4-storage/src/segments/edge/segment.rs b/db4-storage/src/segments/edge/segment.rs index ce937fa437..54e4d8ab0a 100644 --- a/db4-storage/src/segments/edge/segment.rs +++ b/db4-storage/src/segments/edge/segment.rs @@ -16,7 +16,6 @@ use raphtory_api::core::{ entities::{ LayerId, VID, properties::{ - layer_schema::LayerPropSchema, meta::{Meta, STATIC_GRAPH_LAYER_ID}, prop::AsPropRef, }, @@ -134,23 +133,6 @@ impl MemEdgeSegment { self.layers.get(layer_id.0) } - /// Per-layer property-presence schema for this in-memory segment. Returns - /// an empty schema if the layer has no container in this segment (no matching SegmentContainer). - pub fn layer_schema(&self, layer_id: LayerId) -> LayerPropSchema { - self.layers - .get(layer_id.0) - .map(|c| c.layer_schema().clone()) - .unwrap_or_default() - } - - /// Iterator over (layer_id, schema) for every layer present in this in-memory segment. - pub fn layer_schemas(&self) -> impl Iterator { - self.layers - .iter() - .enumerate() - .map(|(i, c)| (LayerId(i), c.layer_schema())) - } - pub fn est_size(&self) -> usize { self.est_size } diff --git a/db4-storage/src/segments/mod.rs b/db4-storage/src/segments/mod.rs index 23fcf4d13e..222c8b2d71 100644 --- a/db4-storage/src/segments/mod.rs +++ b/db4-storage/src/segments/mod.rs @@ -2,7 +2,6 @@ use super::properties::{PropEntry, Properties}; use crate::{LocalPOS, error::StorageError}; use raphtory_api::core::{ entities::properties::{ - layer_schema::LayerPropSchema, meta::Meta, prop::{AsPropRef, Prop}, }, @@ -251,11 +250,6 @@ impl SegmentContainer { &self.properties } - /// Per-layer property-presence summary for this container (which corresponds to a single layer) - pub fn layer_schema(&self) -> &LayerPropSchema { - self.properties.layer_schema() - } - pub fn properties_mut(&mut self) -> &mut Properties { &mut self.properties } diff --git a/db4-storage/src/segments/node/segment.rs b/db4-storage/src/segments/node/segment.rs index 43bf75ca19..d556075f4e 100644 --- a/db4-storage/src/segments/node/segment.rs +++ b/db4-storage/src/segments/node/segment.rs @@ -17,7 +17,6 @@ use raphtory_api::core::{ entities::{ EID, LayerId, VID, properties::{ - layer_schema::LayerPropSchema, meta::Meta, prop::{AsPropRef, Prop}, }, @@ -151,23 +150,6 @@ impl MemNodeSegment { self.layers.get(layer_id.0) } - /// Per-layer property-presence schema for this in-memory segment. Returns - /// an empty schema if the layer has no container in this segment (no matching SegmentContainer). - pub fn layer_schema(&self, layer_id: LayerId) -> LayerPropSchema { - self.layers - .get(layer_id.0) - .map(|c| c.layer_schema().clone()) - .unwrap_or_default() - } - - /// Iterator over (layer_id, schema) for every layer present in this in-memory segment. - pub fn layer_schemas(&self) -> impl Iterator { - self.layers - .iter() - .enumerate() - .map(|(i, c)| (LayerId(i), c.layer_schema())) - } - pub fn degree(&self, n: LocalPOS, layer_id: LayerId, dir: Direction) -> usize { self.get_adj(n, layer_id).map_or(0, |adj| adj.degree(dir)) } diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs index 9faa630c72..b41432a92f 100644 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ b/raphtory-api/src/core/entities/properties/layer_schema.rs @@ -1,9 +1,9 @@ -/// Per-segment / per-layer summary of which property ids have values in a given (layer, segment) pair. -/// The Vec "bitsets" themselves round-trip via rkyv in LayerStatStore when disk storage is enabled. +/// Summary of which property ids have values across one or more layers. /// /// `temporal_props[i] == true` means a value has been written for global -/// temporal-prop-id `i` in this segment for this layer. Same for `metadata` -/// with the global metadata-prop-id space. +/// temporal-prop-id `i`. Same for `metadata` with the global metadata-prop-id +/// space. Built and consumed by the per-layer property presence bitset on +/// `PropMapper` (see `raphtory-api::core::entities::properties::meta`). #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct LayerPropSchema { temporal_props: Vec, @@ -15,15 +15,6 @@ impl LayerPropSchema { Self::default() } - /// Build a schema directly from two presence vectors. Useful when - /// reconstructing a schema from persisted on-disk state. - pub fn from_bools(temporal_props: Vec, metadata: Vec) -> Self { - Self { - temporal_props, - metadata, - } - } - /// Mark a temporal property id as present. #[inline] pub fn insert_temporal(&mut self, prop_id: usize) { @@ -66,26 +57,14 @@ impl LayerPropSchema { .filter_map(|(i, &b)| b.then_some(i)) } - pub fn temporal_bits(&self) -> &[bool] { - &self.temporal_props - } - - pub fn metadata_bits(&self) -> &[bool] { - &self.metadata - } - - /// Union another schema into this one. - pub fn union_with(&mut self, other: &LayerPropSchema) { - union_into(&mut self.temporal_props, &other.temporal_props); - union_into(&mut self.metadata, &other.metadata); - } - - /// Union temporal property presence bits into this schema + /// Union temporal property presence bits into this schema (position-wise + /// OR). Bits beyond the current length grow the backing vec. pub fn union_temporal_with(&mut self, bits: &[bool]) { union_into(&mut self.temporal_props, bits); } - /// Union metadata presence bits into this schema + /// Union metadata presence bits into this schema (position-wise + /// OR). Bits beyond the current length grow the backing vec. pub fn union_metadata_with(&mut self, bits: &[bool]) { union_into(&mut self.metadata, bits); } @@ -122,16 +101,6 @@ fn intersect_into(dst: &mut Vec, mask: &[bool]) { } } -impl FromIterator for LayerPropSchema { - fn from_iter>(iter: I) -> Self { - let mut acc = LayerPropSchema::new(); - for s in iter { - acc.union_with(&s); - } - acc - } -} - #[cfg(test)] mod tests { use super::*; @@ -156,19 +125,18 @@ mod tests { } #[test] - fn union() { - let mut a = LayerPropSchema::new(); - a.insert_temporal(1); - a.insert_metadata(3); + fn union_bits() { + let mut s = LayerPropSchema::new(); + s.insert_temporal(1); + s.insert_metadata(3); - let mut b = LayerPropSchema::new(); - b.insert_temporal(10); - b.insert_metadata(3); - b.insert_metadata(7); + s.union_temporal_with(&[ + false, false, false, false, false, false, false, false, false, false, true, + ]); + s.union_metadata_with(&[false, false, false, false, false, false, false, true]); - a.union_with(&b); - assert_eq!(a.temporal_prop_ids().collect::>(), vec![1, 10]); - assert_eq!(a.metadata_prop_ids().collect::>(), vec![3, 7]); + assert_eq!(s.temporal_prop_ids().collect::>(), vec![1, 10]); + assert_eq!(s.metadata_prop_ids().collect::>(), vec![3, 7]); } #[test] @@ -188,14 +156,4 @@ mod tests { s.intersect_metadata_with(&[false, false, false, true, true]); assert_eq!(s.metadata_prop_ids().collect::>(), vec![3]); } - - #[test] - fn from_bools_roundtrip() { - let mut s = LayerPropSchema::new(); - s.insert_temporal(2); - s.insert_metadata(0); - let rebuilt = - LayerPropSchema::from_bools(s.temporal_bits().to_vec(), s.metadata_bits().to_vec()); - assert_eq!(s, rebuilt); - } } From 27622beda5b990ba7eb347e43f80425712801b20 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 10 Jun 2026 04:53:13 -0400 Subject: [PATCH 09/27] Get rid of the extra logic we added in the schemas to try and use the previous layer property schemas (property presence bitsets) to avoid work. All of it was redundant and didn't save anything. Removed functions that pass these `LayerPropSchema`s around. --- .../src/model/schema/edge_schema.rs | 42 ++++--------------- .../src/model/schema/node_schema.rs | 16 +------ raphtory-storage/src/graph/edges/edges.rs | 21 +--------- raphtory-storage/src/graph/nodes/nodes_ref.rs | 22 +--------- raphtory/src/db/api/properties/internal.rs | 41 +----------------- .../graph/storage_ops/property_schema.rs | 13 +----- .../db/graph/views/property_redacted_graph.rs | 31 +------------- 7 files changed, 16 insertions(+), 170 deletions(-) diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 492c184479..e413723d45 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -7,14 +7,10 @@ use crate::{ use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use itertools::Itertools; use raphtory::{ - db::{ - api::{properties::internal::EdgePropertySchemaOps, view::StaticGraphViewOps}, - graph::edge::EdgeView, - }, + db::{api::view::StaticGraphViewOps, graph::edge::EdgeView}, prelude::*, }; -use raphtory_api::core::entities::properties::{layer_schema::LayerPropSchema, meta::PropMapper}; -use raphtory_storage::layer_ops::InternalLayerOps; +use raphtory_api::core::entities::properties::meta::PropMapper; use std::collections::HashSet; /// Describes edges between a specific pair of node types — the property and @@ -61,11 +57,9 @@ impl EdgeSchema { async fn properties(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let layers = cloned.graph.layer_ids().clone(); - let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); let schema: SchemaAggregate = cloned .edges() - .map(|e| collect_edge_property_schema(e, &layer_schema, false)) + .map(collect_edge_property_schema) .reduce(merge_schemas) .unwrap_or_default(); schema.into_iter().map(|prop| prop.into()).collect_vec() @@ -76,11 +70,9 @@ impl EdgeSchema { async fn metadata(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let layers = cloned.graph.layer_ids().clone(); - let layer_schema = cloned.graph.edge_layer_prop_schema(&layers); let schema: SchemaAggregate = cloned .edges() - .map(|e| collect_edge_metadata_schema(e, &layer_schema, true)) + .map(collect_edge_metadata_schema) .reduce(merge_schemas) .unwrap_or_default(); schema.into_iter().map(|prop| prop.into()).collect_vec() @@ -89,27 +81,11 @@ impl EdgeSchema { } } -fn collect_schema( - props: P, - mapper: &PropMapper, - layer_schema: &LayerPropSchema, - is_metadata: bool, -) -> SchemaAggregate { +fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggregate { props .iter() .zip(props.ids()) .filter_map(|((key, value), id)| { - // skip properties not in the layer schema - // FIXME: Is this even necessary? - // `edge.properties()` should only return properties present somewhere in the graph (by definition) - // For PropertyRedactedGraph, hidden properties are filtered out by InternalTemporalPropertiesOps - // and InternalMetadataOps. I think the layer_schema is useless here. - if (is_metadata && !layer_schema.contains_metadata(id)) - || (!is_metadata && !layer_schema.contains_temporal(id)) - { - return None; - } - let value = value?; let key_with_prop_type = ( key.to_string(), @@ -125,20 +101,16 @@ fn collect_schema( fn collect_edge_property_schema<'graph, G: GraphViewOps<'graph>>( edge: EdgeView, - layer_schema: &LayerPropSchema, - is_metadata: bool, ) -> SchemaAggregate { let props = edge.properties(); let mapper = edge.graph.edge_meta().temporal_prop_mapper(); - collect_schema(props, mapper, layer_schema, is_metadata) + collect_schema(props, mapper) } fn collect_edge_metadata_schema<'graph, G: GraphViewOps<'graph>>( edge: EdgeView, - layer_schema: &LayerPropSchema, - is_metadata: bool, ) -> SchemaAggregate { let props = edge.metadata(); let mapper = edge.graph.edge_meta().metadata_mapper(); - collect_schema(props, mapper, layer_schema, is_metadata) + collect_schema(props, mapper) } diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index ecb92cbd04..6f8e5f61c3 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -64,12 +64,6 @@ impl NodeSchema { .unwrap_or_else(|| DEFAULT_NODE_TYPE.to_string()) } fn properties_inner(&self) -> Vec { - // Property presence schema across every layer. Shouldn't make a difference for normal `GraphStorage`. - // Masks redacted properties from `PropertyRedactedGraph`. - // FIXME: Shouldn't all ids in the PropMapper exist somewhere in the graph, thus be present in the LayerPropSchema? - // FIXME: For the PropertyRedactedGraph, isn't the same filtering already being applied in `node_visible_temporal_prop_ids`? - // FIXME: Do we even wanna do this? - let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_temporal_prop_ids().collect(); let (keys, property_types): (Vec<_>, Vec<_>) = self @@ -78,7 +72,7 @@ impl NodeSchema { .temporal_prop_mapper() .locked() .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id) && layer_schema.contains_temporal(*id)) + .filter(|(id, _, _)| visible.contains(id)) .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); @@ -120,12 +114,6 @@ impl NodeSchema { } fn metadata_inner(&self) -> Vec { - // Property presence schema across every layer. Shouldn't make a difference for normal `GraphStorage`. - // Masks redacted properties from `PropertyRedactedGraph`. - // FIXME: Shouldn't all ids in the PropMapper exist somewhere in the graph, thus be present in the LayerPropSchema? - // FIXME: For the PropertyRedactedGraph, isn't the same filtering already being applied in `node_visible_temporal_prop_ids`? - // FIXME: Do we even wanna do this? - let layer_schema = self.graph.node_layer_prop_schema(&LayerIds::All); let visible: std::collections::HashSet = self.graph.node_visible_metadata_ids().collect(); let (keys, property_types): (Vec<_>, Vec<_>) = self @@ -134,7 +122,7 @@ impl NodeSchema { .metadata_mapper() .locked() .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id) && layer_schema.contains_metadata(*id)) + .filter(|(id, _, _)| visible.contains(id)) .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index daa4e05fc1..64c27c15f6 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -1,8 +1,7 @@ use super::{edge_entry::EdgeStorageEntry, unlocked::UnlockedEdges}; use either::Either; use raphtory_api::core::entities::{ - properties::{layer_schema::LayerPropSchema, meta::STATIC_GRAPH_LAYER_ID}, - LayerId, LayerIds, EID, + properties::meta::STATIC_GRAPH_LAYER_ID, LayerId, LayerIds, EID, }; use raphtory_core::entities::edges::edge_ref::EdgeRef; use rayon::iter::ParallelIterator; @@ -149,24 +148,6 @@ impl<'a> EdgesStorageRef<'a> { } } - /// Union of edge property presence across the supplied layers - pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - let inner = match self { - EdgesStorageRef::Mem(storage) => storage.storage(), - EdgesStorageRef::Unlocked(storage) => storage.storage(), - }; - let meta = inner.edge_meta(); - let num_layers = inner.num_layers(); - let mut schema = LayerPropSchema::new(); - for layer_id in layers.iter(num_layers) { - meta.temporal_prop_mapper() - .with_layer_prop_bits(layer_id, |bits| schema.union_temporal_with(bits)); - meta.metadata_mapper() - .with_layer_prop_bits(layer_id, |bits| schema.union_metadata_with(bits)); - } - schema - } - /// O(1) check via the per-layer bitset cached on `Meta.temporal_prop_mapper`. /// `false` is authoritative — callers can skip column reads for `(layer_id, prop_id)`. pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { diff --git a/raphtory-storage/src/graph/nodes/nodes_ref.rs b/raphtory-storage/src/graph/nodes/nodes_ref.rs index 94a1cf6a34..ef211e0a9c 100644 --- a/raphtory-storage/src/graph/nodes/nodes_ref.rs +++ b/raphtory-storage/src/graph/nodes/nodes_ref.rs @@ -1,8 +1,6 @@ use super::node_ref::NodeStorageRef; use crate::graph::variants::storage_variants3::StorageVariants3; -use raphtory_api::core::entities::{ - properties::layer_schema::LayerPropSchema, LayerId, LayerIds, VID, -}; +use raphtory_api::core::entities::{LayerId, VID}; use rayon::iter::ParallelIterator; use storage::{Extension, ReadLockedNodes}; @@ -47,24 +45,6 @@ impl<'a> NodesStorageEntry<'a> { for_all_variants!(self, nodes => nodes.iter()) } - /// Union of node property presence across the supplied layers. - pub fn layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - let inner = match self { - NodesStorageEntry::Mem(nodes) => nodes.storage(), - NodesStorageEntry::Unlocked(nodes) => nodes.storage(), - }; - let meta = inner.prop_meta(); - let num_layers = inner.num_layers(); - let mut schema = LayerPropSchema::new(); - for layer_id in layers.iter(num_layers) { - meta.temporal_prop_mapper() - .with_layer_prop_bits(layer_id, |bits| schema.union_temporal_with(bits)); - meta.metadata_mapper() - .with_layer_prop_bits(layer_id, |bits| schema.union_metadata_with(bits)); - } - schema - } - /// O(1) check pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { let inner = match self { diff --git a/raphtory/src/db/api/properties/internal.rs b/raphtory/src/db/api/properties/internal.rs index 8bfc4dff09..02e7b221eb 100644 --- a/raphtory/src/db/api/properties/internal.rs +++ b/raphtory/src/db/api/properties/internal.rs @@ -2,11 +2,8 @@ use crate::db::api::view::BoxedLIter; use raphtory_api::{ core::{ entities::{ - properties::{ - layer_schema::LayerPropSchema, - prop::{Prop, PropType}, - }, - LayerId, LayerIds, + properties::prop::{Prop, PropType}, + LayerId, }, storage::{arc_str::ArcStr, timeindex::EventTime}, }, @@ -27,19 +24,6 @@ pub trait NodePropertySchemaOps: Send + Sync { /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn node_visible_metadata_name(&self, id: usize) -> Option; - /// Union of the (segment, layer) property presence bitsets for all node properties present in the layers. - /// Override for behaviour other than "all node properties registered globally". - fn node_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { - let mut schema = LayerPropSchema::new(); - for id in self.node_visible_temporal_prop_ids() { - schema.insert_temporal(id); - } - for id in self.node_visible_metadata_ids() { - schema.insert_metadata(id); - } - schema - } - /// O(1) check: is temporal-prop `prop_id` present on any node in `layer_id`? fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; @@ -58,19 +42,6 @@ pub trait EdgePropertySchemaOps: Send + Sync { /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn edge_visible_metadata_name(&self, id: usize) -> Option; - /// Union of the (segment, layer) property presence bitsets for all edge properties present in the layers. - /// Override for behaviour other than "all edge properties registered globally". - fn edge_layer_prop_schema(&self, _layers: &LayerIds) -> LayerPropSchema { - let mut schema = LayerPropSchema::new(); - for id in self.edge_visible_temporal_prop_ids() { - schema.insert_temporal(id); - } - for id in self.edge_visible_metadata_ids() { - schema.insert_metadata(id); - } - schema - } - /// O(1) check: is temporal-prop `prop_id` present on any edge in `layer_id`? fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; @@ -112,10 +83,6 @@ where self.base().node_visible_metadata_name(id) } #[inline] - fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - self.base().node_layer_prop_schema(layers) - } - #[inline] fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { self.base().node_layer_has_temporal_prop(layer_id, prop_id) } @@ -161,10 +128,6 @@ where fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { self.base().edge_layer_has_metadata(layer_id, prop_id) } - #[inline] - fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - self.base().edge_layer_prop_schema(layers) - } } pub trait InternalTemporalPropertyViewOps { diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index 67543fb80f..cc8697fd49 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -4,10 +4,7 @@ use crate::db::api::{ view::BoxedLIter, }; use raphtory_api::{ - core::{ - entities::{properties::layer_schema::LayerPropSchema, LayerId, LayerIds}, - storage::arc_str::ArcStr, - }, + core::{entities::LayerId, storage::arc_str::ArcStr}, iter::IntoDynBoxed, }; @@ -34,10 +31,6 @@ impl NodePropertySchemaOps for GraphStorage { Some(self.node_meta().metadata_mapper().get_name(id).clone()) } - fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - self.nodes().layer_prop_schema(layers) - } - fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { self.nodes().layer_has_temporal_prop(layer_id, prop_id) } @@ -70,10 +63,6 @@ impl EdgePropertySchemaOps for GraphStorage { Some(self.edge_meta().metadata_mapper().get_name(id).clone()) } - fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - self.edges().layer_prop_schema(layers) - } - fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { self.edges().layer_has_temporal_prop(layer_id, prop_id) } diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index abe33c4d8c..67106af32e 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -15,8 +15,8 @@ use crate::db::api::{ use raphtory_api::{ core::{ entities::{ - properties::{layer_schema::LayerPropSchema, prop::Prop}, - LayerId, LayerIds, + properties::prop::Prop, + LayerId, }, storage::arc_str::ArcStr, }, @@ -263,14 +263,6 @@ impl NodePropertySchemaOps for PropertyRedactedGraph { is_visible(&self.redaction.node_meta_visible, prop_id) && self.graph.node_layer_has_metadata(layer_id, prop_id) } - - fn node_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - mask_schema( - self.graph.node_layer_prop_schema(layers), - &self.redaction.node_props_visible, - &self.redaction.node_meta_visible, - ) - } } impl EdgePropertySchemaOps for PropertyRedactedGraph { @@ -323,25 +315,6 @@ impl EdgePropertySchemaOps for PropertyRedactedGraph { is_visible(&self.redaction.edge_meta_visible, prop_id) && self.graph.edge_layer_has_metadata(layer_id, prop_id) } - - fn edge_layer_prop_schema(&self, layers: &LayerIds) -> LayerPropSchema { - mask_schema( - self.graph.edge_layer_prop_schema(layers), - &self.redaction.edge_props_visible, - &self.redaction.edge_meta_visible, - ) - } -} - -/// Mask out redacted bits from a layer property schema -fn mask_schema( - mut schema: LayerPropSchema, - temporal_visible: &[bool], - metadata_visible: &[bool], -) -> LayerPropSchema { - schema.intersect_temporal_with(temporal_visible); - schema.intersect_metadata_with(metadata_visible); - schema } // Graph-level property redaction: override InternalTemporalPropertiesOps and InternalMetadataOps From 075b9243a1f9dc00235d2cf77aa0d8add26a9a74 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 10 Jun 2026 05:21:55 -0400 Subject: [PATCH 10/27] Get rid of previously added LayerPropSchema and it's uses --- .../core/entities/properties/layer_schema.rs | 159 ------------------ .../src/core/entities/properties/meta.rs | 9 - .../src/core/entities/properties/mod.rs | 1 - .../db/graph/views/property_redacted_graph.rs | 5 +- 4 files changed, 1 insertion(+), 173 deletions(-) delete mode 100644 raphtory-api/src/core/entities/properties/layer_schema.rs diff --git a/raphtory-api/src/core/entities/properties/layer_schema.rs b/raphtory-api/src/core/entities/properties/layer_schema.rs deleted file mode 100644 index b41432a92f..0000000000 --- a/raphtory-api/src/core/entities/properties/layer_schema.rs +++ /dev/null @@ -1,159 +0,0 @@ -/// Summary of which property ids have values across one or more layers. -/// -/// `temporal_props[i] == true` means a value has been written for global -/// temporal-prop-id `i`. Same for `metadata` with the global metadata-prop-id -/// space. Built and consumed by the per-layer property presence bitset on -/// `PropMapper` (see `raphtory-api::core::entities::properties::meta`). -#[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct LayerPropSchema { - temporal_props: Vec, - metadata: Vec, -} - -impl LayerPropSchema { - pub fn new() -> Self { - Self::default() - } - - /// Mark a temporal property id as present. - #[inline] - pub fn insert_temporal(&mut self, prop_id: usize) { - if prop_id >= self.temporal_props.len() { - self.temporal_props.resize(prop_id + 1, false); - } - self.temporal_props[prop_id] = true; - } - - /// Mark a metadata (const) property id as present. - #[inline] - pub fn insert_metadata(&mut self, prop_id: usize) { - if prop_id >= self.metadata.len() { - self.metadata.resize(prop_id + 1, false); - } - self.metadata[prop_id] = true; - } - - #[inline] - pub fn contains_temporal(&self, prop_id: usize) -> bool { - self.temporal_props.get(prop_id).copied().unwrap_or(false) - } - - #[inline] - pub fn contains_metadata(&self, prop_id: usize) -> bool { - self.metadata.get(prop_id).copied().unwrap_or(false) - } - - pub fn temporal_prop_ids(&self) -> impl Iterator + '_ { - self.temporal_props - .iter() - .enumerate() - .filter_map(|(i, &b)| b.then_some(i)) - } - - pub fn metadata_prop_ids(&self) -> impl Iterator + '_ { - self.metadata - .iter() - .enumerate() - .filter_map(|(i, &b)| b.then_some(i)) - } - - /// Union temporal property presence bits into this schema (position-wise - /// OR). Bits beyond the current length grow the backing vec. - pub fn union_temporal_with(&mut self, bits: &[bool]) { - union_into(&mut self.temporal_props, bits); - } - - /// Union metadata presence bits into this schema (position-wise - /// OR). Bits beyond the current length grow the backing vec. - pub fn union_metadata_with(&mut self, bits: &[bool]) { - union_into(&mut self.metadata, bits); - } - - /// Position-wise AND of the temporal-prop bits with the supplied visibility mask. - /// Careful: Bits at positions beyond `mask.len()` are treated as not visible. - pub fn intersect_temporal_with(&mut self, mask: &[bool]) { - intersect_into(&mut self.temporal_props, mask); - } - - /// Position-wise AND of the metadata bits with the supplied visibility mask. - /// Careful: Bits at positions beyond `mask.len()` are treated as not visible. - pub fn intersect_metadata_with(&mut self, mask: &[bool]) { - intersect_into(&mut self.metadata, mask); - } - - pub fn is_empty(&self) -> bool { - !self.temporal_props.iter().any(|&b| b) && !self.metadata.iter().any(|&b| b) - } -} - -fn union_into(dst: &mut Vec, src: &[bool]) { - if src.len() > dst.len() { - dst.resize(src.len(), false); - } - for (d, s) in dst.iter_mut().zip(src.iter()) { - *d |= *s; - } -} - -fn intersect_into(dst: &mut Vec, mask: &[bool]) { - for (i, d) in dst.iter_mut().enumerate() { - *d &= mask.get(i).copied().unwrap_or(false); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn insert_and_query() { - let mut s = LayerPropSchema::new(); - s.insert_temporal(0); - s.insert_temporal(5); - s.insert_metadata(2); - - assert!(s.contains_temporal(0)); - assert!(s.contains_temporal(5)); - assert!(!s.contains_temporal(1)); - assert!(s.contains_metadata(2)); - assert!(!s.contains_metadata(0)); - - let t: Vec = s.temporal_prop_ids().collect(); - assert_eq!(t, vec![0, 5]); - let m: Vec = s.metadata_prop_ids().collect(); - assert_eq!(m, vec![2]); - } - - #[test] - fn union_bits() { - let mut s = LayerPropSchema::new(); - s.insert_temporal(1); - s.insert_metadata(3); - - s.union_temporal_with(&[ - false, false, false, false, false, false, false, false, false, false, true, - ]); - s.union_metadata_with(&[false, false, false, false, false, false, false, true]); - - assert_eq!(s.temporal_prop_ids().collect::>(), vec![1, 10]); - assert_eq!(s.metadata_prop_ids().collect::>(), vec![3, 7]); - } - - #[test] - fn intersect() { - let mut s = LayerPropSchema::new(); - s.insert_temporal(0); - s.insert_temporal(2); - s.insert_temporal(5); - s.insert_metadata(1); - s.insert_metadata(3); - - // Mask shorter than schema — trailing bits clear. - s.intersect_temporal_with(&[true, false, true]); - assert_eq!(s.temporal_prop_ids().collect::>(), vec![0, 2]); - - // Mask covers schema — keep only marked bits. - s.intersect_metadata_with(&[false, false, false, true, true]); - assert_eq!(s.metadata_prop_ids().collect::>(), vec![3]); - } -} diff --git a/raphtory-api/src/core/entities/properties/meta.rs b/raphtory-api/src/core/entities/properties/meta.rs index c2e299d9b0..7215b3dd44 100644 --- a/raphtory-api/src/core/entities/properties/meta.rs +++ b/raphtory-api/src/core/entities/properties/meta.rs @@ -311,15 +311,6 @@ impl PropMapper { ensure_and_set(&mut guard, layer_id.0, prop_id); } - /// Run `f` against the raw `&[bool]` of `layer_id`'s prop presence row, borrowing under a single read lock. - /// `f` can't acquire a write lock on layer_prop_presence or else it will deadlock. - #[inline] - pub fn with_layer_prop_bits(&self, layer_id: LayerId, f: impl FnOnce(&[bool]) -> R) -> R { - let guard = self.layer_prop_presence.read_recursive(); - let bits: &[bool] = guard.get(layer_id.0).map(|v| v.as_slice()).unwrap_or(&[]); - f(bits) - } - pub fn d_types(&self) -> impl Deref> + '_ { self.dtypes.read_recursive() } diff --git a/raphtory-api/src/core/entities/properties/mod.rs b/raphtory-api/src/core/entities/properties/mod.rs index ebba50fdc4..f20788ef0f 100644 --- a/raphtory-api/src/core/entities/properties/mod.rs +++ b/raphtory-api/src/core/entities/properties/mod.rs @@ -1,4 +1,3 @@ -pub mod layer_schema; pub mod meta; pub mod prop; pub mod tprop; diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index 67106af32e..80e58e21e6 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -14,10 +14,7 @@ use crate::db::api::{ }; use raphtory_api::{ core::{ - entities::{ - properties::prop::Prop, - LayerId, - }, + entities::{properties::prop::Prop, LayerId}, storage::arc_str::ArcStr, }, inherit::Base, From 14eba92b2e74b526ac422bb069d47fff48d71a69 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Fri, 12 Jun 2026 05:01:11 -0400 Subject: [PATCH 11/27] run rustfmt --- raphtory-graphql/src/lib.rs | 3 - raphtory-graphql/src/model/graph/filtering.rs | 43 +++++-- raphtory-tests/tests/test_filters.rs | 46 ++++---- raphtory/src/db/api/state/ops/filter.rs | 22 ++-- .../graph/views/filter/model/degree_filter.rs | 105 +++++++++++------- .../src/db/graph/views/filter/model/mod.rs | 3 +- .../views/filter/model/node_filter/mod.rs | 45 +++++--- .../src/python/filter/node_filter_builders.rs | 12 +- 8 files changed, 174 insertions(+), 105 deletions(-) diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 184f6a0049..1a3f2acfde 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -536,7 +536,6 @@ mod graphql_test { graph } - fn degree_graph_with_add_edge_only() -> Graph { let graph = Graph::new(); @@ -646,8 +645,6 @@ mod graphql_test { ); } - - #[tokio::test] async fn test_unique_temporal_properties() { let g = Graph::new(); diff --git a/raphtory-graphql/src/model/graph/filtering.rs b/raphtory-graphql/src/model/graph/filtering.rs index 3033016e1b..2ab6872c9c 100644 --- a/raphtory-graphql/src/model/graph/filtering.rs +++ b/raphtory-graphql/src/model/graph/filtering.rs @@ -1,4 +1,7 @@ -use crate::model::{graph::{node_id::GqlNodeId, property::Value, timeindex::GqlTimeInput}, plugins::operation}; +use crate::model::{ + graph::{node_id::GqlNodeId, property::Value, timeindex::GqlTimeInput}, + plugins::operation, +}; use async_graphql::dynamic::ValueAccessor; use dynamic_graphql::{ internal::{ @@ -7,16 +10,35 @@ use dynamic_graphql::{ Enum, InputObject, OneOfInput, }; use raphtory::{ - db::{api::{state::ops::Degree, view::internal::filtered_edge}, graph::views::filter::model::{ - ComposableFilter, DynFilter, DynView, NoFilter, ViewWrapOps, degree_filter::DegreeFilter, edge_filter::{CompositeEdgeFilter, EdgeFilter}, filter::{Filter, FilterValue}, filter_operator::FilterOperator, graph_filter::GraphFilter, is_active_edge_filter::IsActiveEdge, is_active_node_filter::IsActiveNode, is_deleted_filter::IsDeletedEdge, is_self_loop_filter::IsSelfLoopEdge, is_valid_filter::IsValidEdge, latest_filter::Latest as LatestWrap, layered_filter::Layered, node_filter::{CompositeNodeFilter, NodeFilter}, property_filter::{Op, PropertyFilter, PropertyFilterValue, PropertyRef}, snapshot_filter::{SnapshotAt as SnapshotAtWrap, SnapshotLatest as SnapshotLatestWrap}, windowed_filter::Windowed - }}, + db::{ + api::{state::ops::Degree, view::internal::filtered_edge}, + graph::views::filter::model::{ + degree_filter::DegreeFilter, + edge_filter::{CompositeEdgeFilter, EdgeFilter}, + filter::{Filter, FilterValue}, + filter_operator::FilterOperator, + graph_filter::GraphFilter, + is_active_edge_filter::IsActiveEdge, + is_active_node_filter::IsActiveNode, + is_deleted_filter::IsDeletedEdge, + is_self_loop_filter::IsSelfLoopEdge, + is_valid_filter::IsValidEdge, + latest_filter::Latest as LatestWrap, + layered_filter::Layered, + node_filter::{CompositeNodeFilter, NodeFilter}, + property_filter::{Op, PropertyFilter, PropertyFilterValue, PropertyRef}, + snapshot_filter::{SnapshotAt as SnapshotAtWrap, SnapshotLatest as SnapshotLatestWrap}, + windowed_filter::Windowed, + ComposableFilter, DynFilter, DynView, NoFilter, ViewWrapOps, + }, + }, errors::GraphError, }; -use raphtory_api::core::Direction; use raphtory_api::core::{ entities::{properties::prop::Prop, Layer, GID}, storage::timeindex::{AsTime, EventTime}, utils::time::IntoTime, + Direction, }; use serde::{Deserialize, Serialize}; use std::{ @@ -304,7 +326,6 @@ pub struct PropertyFilterNew { pub where_: PropCondition, } - /// Filters nodes by computed degree with a directional scope. /// /// `DegreeFilterNew` lets callers filter on: @@ -330,8 +351,8 @@ pub enum DegreeDirection { impl From for Direction { fn from(d: DegreeDirection) -> Self { match d { - DegreeDirection::In => Direction::IN, - DegreeDirection::Out => Direction::OUT, + DegreeDirection::In => Direction::IN, + DegreeDirection::Out => Direction::OUT, DegreeDirection::Both => Direction::BOTH, } } @@ -342,7 +363,7 @@ impl From for String { match d { DegreeDirection::In => "in_degree".to_string(), DegreeDirection::Out => "out_degree".to_string(), - DegreeDirection::Both => "degree".to_string(), + DegreeDirection::Both => "degree".to_string(), } } } @@ -1440,8 +1461,8 @@ impl TryFrom for CompositeNodeFilter { direction: core_direction, operator, value, - ops - })) + ops, + })) } GqlNodeFilter::Property(prop) => { let prop_ref = PropertyRef::Property(prop.name.clone()); diff --git a/raphtory-tests/tests/test_filters.rs b/raphtory-tests/tests/test_filters.rs index a4bc92771c..5976a103ba 100644 --- a/raphtory-tests/tests/test_filters.rs +++ b/raphtory-tests/tests/test_filters.rs @@ -1647,32 +1647,38 @@ fn init_edges_graph_with_str_ids_del< mod test_node_filter { -use crate::{ + use crate::{ init_nodes_graph, init_nodes_graph_with_num_ids, init_nodes_graph_with_str_ids, IdentityGraphTransformer, }; + use proptest::proptest; use raphtory::{ - algorithms::alternating_mask::alternating_mask, core::entities::VID, db::{ - api::view::{Filter, filter_ops::NodeSelect}, - graph::{ - views::filter::{ - CreateFilter, model::{ - ComposableFilter, CompositeNodeFilter, NodeViewFilterOps, PropertyFilterFactory, TryAsCompositeFilter, ViewWrapOps, degree_filter::DegreeFilterFactory, node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, property_filter::ops::{ListAggOps, PropertyFilterOps} - } + algorithms::alternating_mask::alternating_mask, + core::entities::VID, + db::{ + api::view::{filter_ops::NodeSelect, Filter}, + graph::views::filter::{ + model::{ + degree_filter::DegreeFilterFactory, + node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, + property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, + ComposableFilter, CompositeNodeFilter, NodeViewFilterOps, + PropertyFilterFactory, TryAsCompositeFilter, ViewWrapOps, }, + CreateFilter, }, - }, errors::GraphError, prelude::{ - AdditionOps, Graph, GraphViewOps, NO_PROPS, NodeFilter, NodeStateOps, NodeViewOps, TimeOps - } + }, + errors::GraphError, + prelude::{ + AdditionOps, Graph, GraphViewOps, IntoProp, NodeFilter, NodeStateOps, NodeViewOps, + TimeOps, NO_PROPS, + }, }; + use raphtory_api::core::{entities::properties::prop::Prop, Direction}; use raphtory_tests::assertions::{ assert_filter_nodes_results, assert_search_nodes_results, assert_select_nodes_results, TestVariants, }; - use raphtory_api::core::{Direction, entities::properties::prop::Prop}; - use raphtory::prelude::IntoProp; - use raphtory::db::graph::views::filter::model::property_filter::ops::ElemQualifierOps; - use proptest::proptest; fn sort_vids(mut vids: Vec) -> Vec { vids.sort(); @@ -1717,7 +1723,8 @@ use crate::{ .map(|n| n.node) .collect::>(); - let expected_filter_nodes = candidates_with_history_after_filtering(graph, expected_select_nodes.clone()); + let expected_filter_nodes = + candidates_with_history_after_filtering(graph, expected_select_nodes.clone()); let filtered_event_graph = graph.filter(filter.clone()).unwrap(); let filtered_event_nodes = sort_vids( @@ -1799,7 +1806,6 @@ use crate::{ graph } - fn degree_graph_with_add_edge_only() -> Graph { let graph = Graph::new(); @@ -1844,7 +1850,6 @@ use crate::{ graph } - // Property-based tests for degree filtering proptest! { #[test] @@ -2033,7 +2038,7 @@ use crate::{ |d| d > threshold as usize && d < (threshold + 5) as usize, &format!("OUT > {} AND OUT < {}", threshold, threshold + 5), ); - } + } #[test] fn prop_degree_filter_or(threshold in 0u64..15) { @@ -2158,7 +2163,7 @@ use crate::{ &format!("OUT is_not_in({}, {})", val1, val2), ); } - } + } #[test] fn test_degree_filter_with_invalid_expressions() { @@ -13021,4 +13026,3 @@ mod test_edge_composite_filter { ); } } - diff --git a/raphtory/src/db/api/state/ops/filter.rs b/raphtory/src/db/api/state/ops/filter.rs index 11c5a90747..25e4b70d47 100644 --- a/raphtory/src/db/api/state/ops/filter.rs +++ b/raphtory/src/db/api/state/ops/filter.rs @@ -2,18 +2,25 @@ use crate::{ db::{ api::{ state::{ - Index, ops::{Const, Degree, IntoDynNodeOp, NodeOp, TypeId} + ops::{Const, Degree, IntoDynNodeOp, NodeOp, TypeId}, + Index, }, view::internal::{GraphView, NodeList}, }, graph::{ create_node_type_filter, - views::filter::model::{FilterOperator, degree_filter::DegreeFilter, filter::{Filter, FilterValue}, node_filter::NodeFilter, property_filter::PropertyFilterValue}, + views::filter::model::{ + degree_filter::DegreeFilter, + filter::{Filter, FilterValue}, + node_filter::NodeFilter, + property_filter::PropertyFilterValue, + FilterOperator, + }, }, }, prelude::{GraphViewOps, PropertyFilter}, }; -use raphtory_api::core::entities::{VID, properties::prop::Prop}; +use raphtory_api::core::entities::{properties::prop::Prop, VID}; use raphtory_core::entities::nodes::node_ref::AsNodeRef; use raphtory_storage::graph::{graph::GraphStorage, nodes::node_storage_ops::NodeStorageOps}; use std::sync::Arc; @@ -225,19 +232,19 @@ impl NodeOp for NodePropertyFilterOp { pub struct NodeDegreeFilterOp { degree: Degree, operator: FilterOperator, - value: PropertyFilterValue + value: PropertyFilterValue, } impl NodeDegreeFilterOp { pub(crate) fn new(graph: G, filter: DegreeFilter) -> Self { let degree = Degree { dir: filter.direction, - view: graph + view: graph, }; Self { degree, operator: filter.operator, - value: filter.value + value: filter.value, } } } @@ -248,7 +255,8 @@ impl NodeOp for NodeDegreeFilterOp { fn apply(&self, storage: &GraphStorage, node: VID) -> Self::Output { let node_degree = self.degree.apply(storage, node); let node_degree_prop = Prop::U64(node_degree as u64); - self.operator.apply_to_property(&self.value, Some(&node_degree_prop)) + self.operator + .apply_to_property(&self.value, Some(&node_degree_prop)) } } diff --git a/raphtory/src/db/graph/views/filter/model/degree_filter.rs b/raphtory/src/db/graph/views/filter/model/degree_filter.rs index 18e5f3d463..ef6bc5c56f 100644 --- a/raphtory/src/db/graph/views/filter/model/degree_filter.rs +++ b/raphtory/src/db/graph/views/filter/model/degree_filter.rs @@ -1,24 +1,35 @@ -use std::collections::HashSet; -use std::sync::Arc; - -use raphtory_api::core::entities::properties::prop::PropType; -use raphtory_api::core::{Direction, entities::properties::prop::Prop}; -use raphtory_core::entities::{VID}; +use std::{collections::HashSet, sync::Arc}; + +use crate::{ + db::{ + api::{ + state::ops::{filter::NodeDegreeFilterOp, GraphView}, + view::{GraphViewOps, NodeViewOps}, + }, + graph::views::filter::{ + model, + model::{ + property_filter::{ + builders::{PropertyExprBuilder, PropertyExprBuilderInput}, + Op, PropertyFilter, PropertyFilterInput, PropertyFilterValue, PropertyRef, + }, + CombinedFilter, ComposableFilter, CompositeNodeFilter, EntityMarker, + FilterOperator, InternalPropertyFilterBuilder, NodeFilter, TryAsCompositeFilter, + }, + node_filtered_graph::NodeFilteredGraph, + CreateFilter, + }, + }, + errors::GraphError, +}; +use raphtory_api::core::{ + entities::properties::prop::{Prop, PropType}, + Direction, +}; +use raphtory_core::entities::VID; use raphtory_storage::graph::nodes::{node_ref::NodeStorageRef, node_storage_ops::NodeStorageOps}; -use crate::db::api::state::ops::GraphView; -use crate::db::api::state::ops::filter::NodeDegreeFilterOp; -use crate::db::graph::views::filter::CreateFilter; -use crate::db::graph::views::filter::model::{ComposableFilter, CompositeNodeFilter, NodeFilter}; -use crate::db::graph::views::filter::model::property_filter::{Op, PropertyFilterInput, PropertyRef, PropertyFilter}; -use crate::db::graph::views::filter::model::property_filter::builders::{PropertyExprBuilder, PropertyExprBuilderInput}; -use crate::db::graph::views::filter::model::{CombinedFilter, EntityMarker, InternalPropertyFilterBuilder, TryAsCompositeFilter}; -use crate::db::graph::views::filter::model; -use crate::db::graph::views::filter::node_filtered_graph::NodeFilteredGraph; -use crate::db::{api::view::{GraphViewOps, NodeViewOps}, graph::views::filter::model::{FilterOperator, property_filter::PropertyFilterValue}}; -use crate::errors::GraphError; use std::{fmt, fmt::Display}; - #[derive(Clone)] pub struct DegreeFilterBuilder { direction: Direction, @@ -42,7 +53,6 @@ pub struct DegreeFilter { pub ops: Vec, } - impl CreateFilter for DegreeFilter { type EntityFiltered<'graph, G: GraphViewOps<'graph>> = NodeFilteredGraph>; @@ -73,22 +83,30 @@ impl CreateFilter for DegreeFilter { )); } match self.operator { - FilterOperator::Eq | FilterOperator::Ne| FilterOperator::Gt | FilterOperator::Ge | FilterOperator::Lt | FilterOperator::Le | FilterOperator::IsIn | FilterOperator::IsNotIn => {}, + FilterOperator::Eq + | FilterOperator::Ne + | FilterOperator::Gt + | FilterOperator::Ge + | FilterOperator::Lt + | FilterOperator::Le + | FilterOperator::IsIn + | FilterOperator::IsNotIn => {} _ => { - return Err(GraphError::InvalidFilter( - format!("degree filter does not support operator {:?}", self.operator) - )); + return Err(GraphError::InvalidFilter(format!( + "degree filter does not support operator {:?}", + self.operator + ))); } } let value = match self.value { PropertyFilterValue::Single(ref prop_val) => { let casted_val = prop_val.clone().try_cast(PropType::U64).ok_or_else(|| { GraphError::InvalidFilter(format!( - "degree filter expects an integer value, got {}", + "degree filter expects an integer value, got {}", prop_val.to_string() )) })?; - + PropertyFilterValue::Single(casted_val) } PropertyFilterValue::Set(ref prop_vals) => { @@ -97,7 +115,7 @@ impl CreateFilter for DegreeFilter { .map(|val| { val.clone().try_cast(PropType::U64).ok_or_else(|| { GraphError::InvalidFilter(format!( - "degree filter expects an integer value, got {}", + "degree filter expects an integer value, got {}", val.to_string() )) }) @@ -108,11 +126,11 @@ impl CreateFilter for DegreeFilter { } PropertyFilterValue::None => { return Err(GraphError::InvalidFilter( - "degree filter requires a value".to_string() + "degree filter requires a value".to_string(), )); } - }; - let mut filter = self.clone(); + }; + let mut filter = self.clone(); filter.value = value; Ok(NodeDegreeFilterOp::new(graph, filter)) } @@ -126,19 +144,20 @@ impl CreateFilter for DegreeFilter { } impl TryAsCompositeFilter for DegreeFilter { - fn try_as_composite_edge_filter(&self) -> Result { - Err(GraphError::NotSupported) + fn try_as_composite_edge_filter( + &self, + ) -> Result { + Err(GraphError::NotSupported) } fn try_as_composite_exploded_edge_filter( &self, - ) -> Result - { - Err(GraphError::NotSupported) - } + ) -> Result { + Err(GraphError::NotSupported) + } fn try_as_composite_node_filter(&self) -> Result { Ok(CompositeNodeFilter::Degree(self.clone())) } -} +} fn property_ref(direction: &Direction) -> PropertyRef { match direction { @@ -150,7 +169,7 @@ fn property_ref(direction: &Direction) -> PropertyRef { impl InternalPropertyFilterBuilder for DegreeFilterBuilder where - DegreeFilter: CombinedFilter + DegreeFilter: CombinedFilter, { type Filter = DegreeFilter; type ExprBuilder = DegreeFilterBuilder; @@ -170,10 +189,10 @@ where fn filter(&self, filter: PropertyFilterInput) -> Self::Filter { DegreeFilter { - value: filter.prop_value, - direction: self.direction, - operator: filter.operator, - ops: filter.ops, + value: filter.prop_value, + direction: self.direction, + operator: filter.operator, + ops: filter.ops, } } @@ -187,7 +206,7 @@ where impl ComposableFilter for DegreeFilter {} pub trait DegreeFilterFactory { - fn in_degree(&self) -> DegreeFilterBuilder; + fn in_degree(&self) -> DegreeFilterBuilder; fn out_degree(&self) -> DegreeFilterBuilder; fn degree(&self) -> DegreeFilterBuilder; } @@ -203,4 +222,4 @@ impl Display for DegreeFilter { }; property_filter.fmt(f) } -} +} diff --git a/raphtory/src/db/graph/views/filter/model/mod.rs b/raphtory/src/db/graph/views/filter/model/mod.rs index ef20e4609e..b95369693f 100644 --- a/raphtory/src/db/graph/views/filter/model/mod.rs +++ b/raphtory/src/db/graph/views/filter/model/mod.rs @@ -56,6 +56,7 @@ use raphtory_api::core::{ use std::{ops::Deref, sync::Arc}; pub mod and_filter; +pub mod degree_filter; pub mod edge_filter; pub mod exploded_edge_filter; pub mod filter; @@ -75,8 +76,6 @@ pub mod or_filter; pub mod property_filter; pub mod snapshot_filter; pub mod windowed_filter; -pub mod degree_filter; - #[derive(Debug, Copy, Clone)] pub struct NoFilter; diff --git a/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs b/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs index 4dd4cc38fd..85bf52f82e 100644 --- a/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs +++ b/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs @@ -1,31 +1,47 @@ use crate::{ + api::core::Direction, db::{ api::{ state::{ - NodeStateValue, TypedNodeState, ops::{ - NodeOp, TypeId, filter::{ + ops::{ + filter::{ AndOp, MaskOp, NodeIdFilterOp, NodeNameFilterOp, NodeTypeFilterOp, NotOp, OrOp, - } - } + }, + NodeOp, TypeId, + }, + NodeStateValue, TypedNodeState, }, - view::{BoxableGraphView, internal::GraphView}, + view::{internal::GraphView, BoxableGraphView}, }, graph::views::filter::{ - CreateFilter, model::{ - AndFilter, CombinedFilter, ComposableFilter, CompositeExplodedEdgeFilter, EntityMarker, InternalPropertyFilterFactory, InternalViewWrapOps, NodeViewFilterOps, NotFilter, OrFilter, TryAsCompositeFilter, Wrap, degree_filter::{DegreeFilter, DegreeFilterBuilder}, edge_filter::CompositeEdgeFilter, filter::Filter, is_active_node_filter::IsActiveNode, latest_filter::Latest, layered_filter::Layered, node_filter::{ + model::{ + degree_filter::{DegreeFilter, DegreeFilterBuilder, DegreeFilterFactory}, + edge_filter::CompositeEdgeFilter, + filter::Filter, + is_active_node_filter::IsActiveNode, + latest_filter::Latest, + layered_filter::Layered, + node_filter::{ builders::{NodeIdFilterBuilder, NodeNameFilterBuilder, NodeTypeFilterBuilder}, validate::validate, - }, node_state_filter::NodeStateBoolColOp, property_filter::builders::{MetadataFilterBuilder, PropertyFilterBuilder}, snapshot_filter::{SnapshotAt, SnapshotLatest}, windowed_filter::Windowed - }, node_filtered_graph::NodeFilteredGraph + }, + node_state_filter::NodeStateBoolColOp, + property_filter::builders::{MetadataFilterBuilder, PropertyFilterBuilder}, + snapshot_filter::{SnapshotAt, SnapshotLatest}, + windowed_filter::Windowed, + AndFilter, CombinedFilter, ComposableFilter, CompositeExplodedEdgeFilter, + EntityMarker, InternalPropertyFilterFactory, InternalViewWrapOps, + NodeViewFilterOps, NotFilter, OrFilter, TryAsCompositeFilter, Wrap, + }, + node_filtered_graph::NodeFilteredGraph, + CreateFilter, }, }, errors::GraphError, prelude::{GraphViewOps, PropertyFilter}, }; use raphtory_api::core::storage::timeindex::EventTime; -use crate::api::core::Direction; -use crate::db::graph::views::filter::model::degree_filter::DegreeFilterFactory; use std::{fmt, fmt::Display, sync::Arc}; pub mod builders; @@ -106,19 +122,18 @@ impl InternalPropertyFilterFactory for NodeFilter { impl DegreeFilterFactory for NodeFilter { fn degree(&self) -> DegreeFilterBuilder { - DegreeFilterBuilder::new(Direction::BOTH) + DegreeFilterBuilder::new(Direction::BOTH) } fn in_degree(&self) -> DegreeFilterBuilder { - DegreeFilterBuilder::new(Direction::IN) + DegreeFilterBuilder::new(Direction::IN) } fn out_degree(&self) -> DegreeFilterBuilder { - DegreeFilterBuilder::new(Direction::OUT) + DegreeFilterBuilder::new(Direction::OUT) } } - impl NodeViewFilterOps for NodeFilter { type Output = T; diff --git a/raphtory/src/python/filter/node_filter_builders.rs b/raphtory/src/python/filter/node_filter_builders.rs index 84317343c8..1fd4147f11 100644 --- a/raphtory/src/python/filter/node_filter_builders.rs +++ b/raphtory/src/python/filter/node_filter_builders.rs @@ -1,8 +1,14 @@ use crate::{ db::graph::views::filter::model::{ - NodeViewFilterOps, PropertyFilterFactory, ViewWrapOps, degree_filter::DegreeFilterFactory, node_filter::{ - NodeFilter, builders::{NodeIdFilterBuilder, NodeNameFilterBuilder, NodeTypeFilterBuilder}, ops::{NodeFilterOps, NodeIdFilterOps} - }, node_state_filter::NodeStateBoolColOp, property_filter::builders::{MetadataFilterBuilder, PropertyFilterBuilder} + degree_filter::DegreeFilterFactory, + node_filter::{ + builders::{NodeIdFilterBuilder, NodeNameFilterBuilder, NodeTypeFilterBuilder}, + ops::{NodeFilterOps, NodeIdFilterOps}, + NodeFilter, + }, + node_state_filter::NodeStateBoolColOp, + property_filter::builders::{MetadataFilterBuilder, PropertyFilterBuilder}, + NodeViewFilterOps, PropertyFilterFactory, ViewWrapOps, }, python::{ filter::{ From ba4aaff755613a3d87164be6ca3ac0fd0a7481e6 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Mon, 15 Jun 2026 05:50:41 -0400 Subject: [PATCH 12/27] Change LayerSchema to scan edges once and bucket them by (src_node_type, dst_node_type). Introduce limits for max number of edges/values in properties and edges. --- .../src/model/schema/edge_schema.rs | 79 +++++++++++++------ .../src/model/schema/layer_schema.rs | 42 +++++++--- raphtory-graphql/src/model/schema/mod.rs | 5 ++ .../src/model/schema/node_schema.rs | 8 +- 4 files changed, 98 insertions(+), 36 deletions(-) diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index e413723d45..2088256aec 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -1,6 +1,7 @@ use crate::{ model::schema::{ - get_node_type, merge_schemas, property_schema::PropertySchema, SchemaAggregate, + merge_schemas, property_schema::PropertySchema, SchemaAggregate, + MAX_DETAILED_SCHEMA_ENTITIES, }, rayon::blocking_compute, }; @@ -10,8 +11,10 @@ use raphtory::{ db::{api::view::StaticGraphViewOps, graph::edge::EdgeView}, prelude::*, }; -use raphtory_api::core::entities::properties::meta::PropMapper; -use std::collections::HashSet; +use raphtory_api::core::entities::{ + edges::edge_ref::EdgeRef, properties::meta::PropMapper, LayerIds, +}; +use std::{collections::HashSet, sync::Arc}; /// Describes edges between a specific pair of node types — the property and /// metadata keys seen on such edges, along with their observed value types. @@ -21,23 +24,22 @@ pub(crate) struct EdgeSchema { graph: G, src_type: String, dst_type: String, + // scan once and remember edges matching the (srcType, dstType) + edges: Arc<[EdgeRef]>, } impl EdgeSchema { - pub fn new(graph: G, src_type: String, dst_type: String) -> Self { + pub fn new(graph: G, src_type: String, dst_type: String, edges: Vec) -> Self { Self { graph, src_type, dst_type, + edges: edges.into(), } } - fn edges(&self) -> impl Iterator> { - (&&self.graph).edges().into_iter().filter(|&edge| { - let src_type = get_node_type(edge.src()); - let dst_type = get_node_type(edge.dst()); - src_type == self.src_type && dst_type == self.dst_type - }) + fn edges(&self) -> impl Iterator> + '_ { + self.edges.iter().map(|e| EdgeView::new(&self.graph, *e)) } } @@ -57,12 +59,29 @@ impl EdgeSchema { async fn properties(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let schema: SchemaAggregate = cloned - .edges() - .map(collect_edge_property_schema) - .reduce(merge_schemas) - .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { + // large graph, do not collect detailed schema as it is expensive + let visible: HashSet = + cloned.graph.edge_visible_temporal_prop_ids().collect(); + cloned + .graph + .edge_meta() + .temporal_prop_mapper() + .locked() + .iter_ids_and_types() + .filter(|(id, _, _)| visible.contains(id)) + .map(|(_, name, dtype)| { + PropertySchema::new(name.to_string(), dtype.to_string(), vec![]) + }) + .collect() + } else { + let schema: SchemaAggregate = cloned + .edges() + .map(collect_edge_property_schema) + .reduce(merge_schemas) + .unwrap_or_default(); + schema.into_iter().map(|prop| prop.into()).collect_vec() + } }) .await } @@ -70,12 +89,28 @@ impl EdgeSchema { async fn metadata(&self) -> Vec { let cloned = self.clone(); blocking_compute(move || { - let schema: SchemaAggregate = cloned - .edges() - .map(collect_edge_metadata_schema) - .reduce(merge_schemas) - .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { + // large graph, do not collect detailed schema as it is expensive + let visible: HashSet = cloned.graph.edge_visible_metadata_ids().collect(); + cloned + .graph + .edge_meta() + .metadata_mapper() + .locked() + .iter_ids_and_types() + .filter(|(id, _, _)| visible.contains(id)) + .map(|(_, name, dtype)| { + PropertySchema::new(name.to_string(), dtype.to_string(), vec![]) + }) + .collect() + } else { + let schema: SchemaAggregate = cloned + .edges() + .map(collect_edge_metadata_schema) + .reduce(merge_schemas) // FIXME: Stop scanning all properties, take the first (or last) 20 + .unwrap_or_default(); + schema.into_iter().map(|prop| prop.into()).collect_vec() + } }) .await } diff --git a/raphtory-graphql/src/model/schema/layer_schema.rs b/raphtory-graphql/src/model/schema/layer_schema.rs index 3cfbfa5c3e..7ac7a70b26 100644 --- a/raphtory-graphql/src/model/schema/layer_schema.rs +++ b/raphtory-graphql/src/model/schema/layer_schema.rs @@ -1,10 +1,14 @@ -use crate::model::schema::{edge_schema::EdgeSchema, get_node_type}; +use crate::{ + model::schema::{edge_schema::EdgeSchema, get_node_type, MAX_DETAILED_SCHEMA_ENTITIES}, + rayon::blocking_compute, +}; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; -use itertools::Itertools; use raphtory::{ db::{api::view::StaticGraphViewOps, graph::views::layer_graph::LayeredGraph}, prelude::*, }; +use raphtory_api::core::entities::edges::edge_ref::EdgeRef; +use std::collections::HashMap; /// Describes a single edge layer — its name and the per `(srcType, dstType)` /// edge schemas observed within it. @@ -33,16 +37,32 @@ impl LayerSchema { } /// Returns the list of edge schemas for this edge layer async fn edges(&self) -> Vec>> { - self.graph - .edges() - .into_iter() - .map(|edge| { + let graph = self.graph.clone(); + blocking_compute(move || { + // Single scan over the layer's edges, bucketing them by (src_node_type, dst_node_type) + let mut buckets: HashMap<(String, String), Vec> = HashMap::new(); + let mut total = 0usize; + for edge in graph.edges().into_iter() { + // FIXME: Do we stop if we have over 1000 edges or no? + // total += 1; + // if total > MAX_DETAILED_SCHEMA_ENTITIES { + // // Too many edges to build a detailed schema; abort + // return vec![]; + // } let src_type = get_node_type(edge.src()); let dst_type = get_node_type(edge.dst()); - (src_type, dst_type) - }) - .unique() - .map(|(src_type, dst_type)| EdgeSchema::new(self.graph.clone(), src_type, dst_type)) - .collect_vec() + buckets + .entry((src_type, dst_type)) + .or_default() + .push(edge.edge); + } + buckets + .into_iter() + .map(|((src_type, dst_type), edges)| { + EdgeSchema::new(graph.clone(), src_type, dst_type, edges) + }) + .collect() + }) + .await } } diff --git a/raphtory-graphql/src/model/schema/mod.rs b/raphtory-graphql/src/model/schema/mod.rs index 877fb8d1c7..1e4525d859 100644 --- a/raphtory-graphql/src/model/schema/mod.rs +++ b/raphtory-graphql/src/model/schema/mod.rs @@ -13,6 +13,11 @@ pub(crate) mod property_schema; const ENUM_BOUNDARY: usize = 20; +/// Above this many entities (nodes graph-wide for `NodeSchema`, edges +/// graph-wide for `EdgeSchema`), schema resolvers skip collecting property +/// values (variants) and only return keys and types +const MAX_DETAILED_SCHEMA_ENTITIES: usize = 1000; + const DEFAULT_NODE_TYPE: &'static str = "None"; fn get_node_type<'graph, G: GraphViewOps<'graph>>(node: NodeView<'graph, G>) -> String { diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 6f8e5f61c3..2ce583ee2c 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -1,4 +1,6 @@ -use crate::model::schema::{property_schema::PropertySchema, DEFAULT_NODE_TYPE}; +use crate::model::schema::{ + property_schema::PropertySchema, DEFAULT_NODE_TYPE, MAX_DETAILED_SCHEMA_ENTITIES, +}; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use raphtory::{ db::{ @@ -76,7 +78,7 @@ impl NodeSchema { .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); - if self.graph.unfiltered_num_nodes(&LayerIds::All) > 1000 { + if self.graph.unfiltered_num_nodes(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { // large graph, do not collect detailed schema as it is expensive keys.into_iter() .zip(property_types) @@ -126,7 +128,7 @@ impl NodeSchema { .map(|(_, name, dtype)| (name.to_string(), dtype.to_string())) .unzip(); - if self.graph.unfiltered_num_nodes(&LayerIds::All) > 1000 { + if self.graph.unfiltered_num_nodes(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { // large graph, do not collect detailed schema as it is expensive keys.into_iter() .zip(property_types) From 0da92e9c46c24e5188a501bb87ba8bfbc923c787 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 16 Jun 2026 04:57:09 -0400 Subject: [PATCH 13/27] Cleanup after merge --- raphtory-storage/src/graph/nodes/node_storage_ops.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/raphtory-storage/src/graph/nodes/node_storage_ops.rs b/raphtory-storage/src/graph/nodes/node_storage_ops.rs index 71fbb569c0..6ab44a792d 100644 --- a/raphtory-storage/src/graph/nodes/node_storage_ops.rs +++ b/raphtory-storage/src/graph/nodes/node_storage_ops.rs @@ -2,9 +2,7 @@ use crate::graph::layer_ids_with_static; use raphtory_api::core::{ entities::{ edges::edge_ref::EdgeRef, - properties::{ - prop::Prop, - }, + properties::{meta::STATIC_GRAPH_LAYER, prop::Prop}, GidRef, LayerId, LayerIds, VID, }, storage::timeindex::TimeIndexOps, From 733194bc6d4178cccefdad1fc0b94694430de245 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 16 Jun 2026 06:32:41 -0400 Subject: [PATCH 14/27] Update schema collection on edge properties/metadata to avoid scanning all properties. As soon as a property key has more than ENUM_BOUNDARY values, we skip property value collection. --- .../src/model/schema/edge_schema.rs | 83 ++++++++++--------- raphtory-graphql/src/model/schema/mod.rs | 19 +---- 2 files changed, 45 insertions(+), 57 deletions(-) diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 2088256aec..f193650156 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -1,12 +1,11 @@ use crate::{ model::schema::{ - merge_schemas, property_schema::PropertySchema, SchemaAggregate, + property_schema::PropertySchema, SchemaAggregate, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES, }, rayon::blocking_compute, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; -use itertools::Itertools; use raphtory::{ db::{api::view::StaticGraphViewOps, graph::edge::EdgeView}, prelude::*, @@ -14,7 +13,10 @@ use raphtory::{ use raphtory_api::core::entities::{ edges::edge_ref::EdgeRef, properties::meta::PropMapper, LayerIds, }; -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{hash_map::Entry, HashSet}, + sync::Arc, +}; /// Describes edges between a specific pair of node types — the property and /// metadata keys seen on such edges, along with their observed value types. @@ -75,12 +77,11 @@ impl EdgeSchema { }) .collect() } else { - let schema: SchemaAggregate = cloned - .edges() - .map(collect_edge_property_schema) - .reduce(merge_schemas) - .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + let meta = cloned.graph.edge_meta(); + collect_schema( + cloned.edges().map(|edge| edge.properties()), + meta.temporal_prop_mapper(), + ) } }) .await @@ -104,24 +105,26 @@ impl EdgeSchema { }) .collect() } else { - let schema: SchemaAggregate = cloned - .edges() - .map(collect_edge_metadata_schema) - .reduce(merge_schemas) // FIXME: Stop scanning all properties, take the first (or last) 20 - .unwrap_or_default(); - schema.into_iter().map(|prop| prop.into()).collect_vec() + let meta = cloned.graph.edge_meta(); + collect_schema( + cloned.edges().map(|edge| edge.metadata()), + meta.metadata_mapper(), + ) } }) .await } } -fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggregate { - props - .iter() - .zip(props.ids()) - .filter_map(|((key, value), id)| { - let value = value?; +/// Aggregate `(key, dtype) -> distinct values` across all edges, capping each value set at `ENUM_BOUNDARY` +fn collect_schema( + props_per_edge: impl Iterator, + mapper: &PropMapper, +) -> Vec { + let mut schema = SchemaAggregate::default(); + for props in props_per_edge { + for ((key, value), id) in props.iter().zip(props.ids()) { + let Some(value) = value else { continue }; let key_with_prop_type = ( key.to_string(), mapper @@ -129,23 +132,23 @@ fn collect_schema(props: P, mapper: &PropMapper) -> SchemaAggr .expect("type for internal id should always exist") .to_string(), ); - Some((key_with_prop_type, HashSet::from([value.to_string()]))) - }) - .collect() -} - -fn collect_edge_property_schema<'graph, G: GraphViewOps<'graph>>( - edge: EdgeView, -) -> SchemaAggregate { - let props = edge.properties(); - let mapper = edge.graph.edge_meta().temporal_prop_mapper(); - collect_schema(props, mapper) -} - -fn collect_edge_metadata_schema<'graph, G: GraphViewOps<'graph>>( - edge: EdgeView, -) -> SchemaAggregate { - let props = edge.metadata(); - let mapper = edge.graph.edge_meta().metadata_mapper(); - collect_schema(props, mapper) + match schema.entry(key_with_prop_type) { + Entry::Vacant(entry) => { + entry.insert(HashSet::from([value.to_string()])); + } + Entry::Occupied(mut entry) => { + let variants = entry.get_mut(); + // An empty set means "too many variants" so we skip. + // Otherwise, there should always be at least 1 value in the set. + if !variants.is_empty() { + variants.insert(value.to_string()); + if variants.len() > ENUM_BOUNDARY { + variants.clear(); + } + } + } + } + } + } + schema.into_iter().map(|prop| prop.into()).collect() } diff --git a/raphtory-graphql/src/model/schema/mod.rs b/raphtory-graphql/src/model/schema/mod.rs index 1e4525d859..b7377e70ed 100644 --- a/raphtory-graphql/src/model/schema/mod.rs +++ b/raphtory-graphql/src/model/schema/mod.rs @@ -27,21 +27,6 @@ fn get_node_type<'graph, G: GraphViewOps<'graph>>(node: NodeView<'graph, G>) -> } } +/// Maps each `(property key, property type)` to its distinct values. +/// An empty HashSet means "too many values" so we're skipping value collection. type SchemaAggregate = FxHashMap<(String, String), HashSet>; - -fn merge_schemas(mut s1: SchemaAggregate, s2: SchemaAggregate) -> SchemaAggregate { - for ((key, prop_type), set2) in s2 { - if let Some(set1) = s1.get_mut(&(key.clone(), prop_type.clone())) { - // Here, an empty set means: too many values to be interpreted as an enumerated type - if set1.len() > 0 && set2.len() > 0 { - set1.extend(set2); - } - if set1.len() > ENUM_BOUNDARY { - set1.clear(); - } - } else { - s1.insert((key, prop_type), set2); - } - } - s1 -} From 5d659aaa4a2654721b4052f69e98ab8e8381d3dd Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Thu, 18 Jun 2026 04:03:21 -0400 Subject: [PATCH 15/27] Added a SchemaCache to allow caching of generated EdgeSchemas properties and metadata. This is held by GraphWithVectors(Inner). It gets passed to GqlGraph for base/unfiltered (e.g. materialized) graphs, which passes them to EdgeSchema by Arc. --- raphtory-graphql/src/data.rs | 22 ++++++-- raphtory-graphql/src/graph.rs | 16 ++++++ raphtory-graphql/src/model/graph/graph.rs | 27 +++++++++- .../src/model/graph/mutable_graph.rs | 3 ++ raphtory-graphql/src/model/mod.rs | 4 +- raphtory-graphql/src/model/schema/cache.rs | 44 +++++++++++++++ .../src/model/schema/edge_schema.rs | 54 ++++++++++++++++--- .../src/model/schema/graph_schema.rs | 11 ++-- .../src/model/schema/layer_schema.rs | 29 +++++++--- raphtory-graphql/src/model/schema/mod.rs | 1 + .../src/model/schema/node_schema.rs | 2 +- .../src/model/schema/property_schema.rs | 2 +- 12 files changed, 188 insertions(+), 27 deletions(-) create mode 100644 raphtory-graphql/src/model/schema/cache.rs diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index b7fef8aacd..6e08cc25dd 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -12,6 +12,7 @@ use crate::{ namespaced_item::NamespacedItem, vectorised_graph::GqlVectorisedGraph, }, + schema::cache::SchemaCache, }, paths::{ mark_dirty, ExistingGraphFolder, InternalPathValidationError, PathValidationError, @@ -653,11 +654,16 @@ impl Data { path: &str, perm: GraphPermission, graph_type: Option, - ) -> async_graphql::Result<(ExistingGraphFolder, DynamicGraph)> { + ) -> async_graphql::Result<(ExistingGraphFolder, DynamicGraph, Option>)> { let gwv = self.get_graph(path).await?; + // Cache can't be used if we change the graph type; a type override produces a different view + let mut cacheable = graph_type.is_none(); let typed_graph = match graph_type { Some(GqlGraphType::Event) => match gwv.graph() { - MaterializedGraph::EventGraph(g) => MaterializedGraph::EventGraph(g.clone()), + MaterializedGraph::EventGraph(g) => { + cacheable = true; + MaterializedGraph::EventGraph(g.clone()) + } MaterializedGraph::PersistentGraph(g) => { MaterializedGraph::EventGraph(g.event_graph()) } @@ -667,6 +673,7 @@ impl Data { MaterializedGraph::PersistentGraph(g.persistent_graph()) } MaterializedGraph::PersistentGraph(g) => { + cacheable = true; MaterializedGraph::PersistentGraph(g.clone()) } }, @@ -677,11 +684,14 @@ impl Data { filter: Some(ref f), } = perm { + // a redacted view hides keys, and a cached schema could leak redacted information/properties + cacheable = false; apply_access_filter(raw, f).await? } else { raw }; - Ok((gwv.folder().clone(), graph)) + let schema_cache = cacheable.then(|| gwv.schema_cache()); + Ok((gwv.folder().clone(), graph, schema_cache)) } /// For the `graph()` resolver: permission denial → `Ok(None)` (null to client, hides @@ -691,7 +701,8 @@ impl Data { ctx: &Context<'_>, path: &str, graph_type: Option, - ) -> async_graphql::Result> { + ) -> async_graphql::Result>)>> + { match require_at_least_read(ctx, &self.auth_policy, path) { Ok(perm) => self.load_and_filter(path, perm, graph_type).await.map(Some), Err(_) => Ok(None), @@ -709,7 +720,8 @@ impl Data { graph_type: Option, ) -> async_graphql::Result<(ExistingGraphFolder, DynamicGraph)> { let perm = require_at_least_read(ctx, &self.auth_policy, path)?; - self.load_and_filter(path, perm, graph_type).await + let (folder, graph, _cache) = self.load_and_filter(path, perm, graph_type).await?; + Ok((folder, graph)) } /// Checks read permission then returns the raw `GraphWithVectors` (unfiltered). diff --git a/raphtory-graphql/src/graph.rs b/raphtory-graphql/src/graph.rs index abe8a4217f..2d1ba10171 100644 --- a/raphtory-graphql/src/graph.rs +++ b/raphtory-graphql/src/graph.rs @@ -1,4 +1,5 @@ use crate::{ + model::schema::cache::SchemaCache, paths::{ExistingGraphFolder, ValidGraphPaths}, rayon::blocking_compute, }; @@ -46,6 +47,9 @@ pub struct GraphWithVectorsInner { pub folder: ExistingGraphFolder, pub is_dirty: AtomicBool, pub is_flushing: AtomicBool, + /// Cache of computed edge schemas for the unfiltered base view of this graph. + /// Cleared on every mutation + pub schema_cache: Arc, } impl GraphWithVectors { @@ -60,6 +64,7 @@ impl GraphWithVectors { folder, is_dirty: AtomicBool::new(false), is_flushing: AtomicBool::new(false), + schema_cache: Arc::new(SchemaCache::new()), }); Self { inner } } @@ -96,6 +101,17 @@ impl GraphWithVectors { pub fn folder(&self) -> &ExistingGraphFolder { &self.inner.folder } + + /// Handle to this graph's schema cache. + pub fn schema_cache(&self) -> Arc { + self.inner.schema_cache.clone() + } + + /// Drop all cached schemas. Called after every mutation. + pub fn invalidate_schema_cache(&self) { + self.inner.schema_cache.invalidate(); + } + pub fn set_dirty(&self, is_dirty: bool) { self.inner.is_dirty.store(is_dirty, Ordering::Release); } diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index 25bf609365..5921ff8217 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -16,7 +16,7 @@ use crate::{ GqlAlignmentUnit, WindowDuration, }, plugins::graph_algorithm_plugin::GraphAlgorithmPlugin, - schema::graph_schema::GraphSchema, + schema::{cache::SchemaCache, graph_schema::GraphSchema}, }, paths::{ExistingGraphFolder, PathValidationError, ValidGraphPaths}, rayon::blocking_compute, @@ -62,6 +62,9 @@ use std::{ pub(crate) struct GqlGraph { path: ExistingGraphFolder, graph: DynamicGraph, + // Shared schema cache, set only for the unfiltered native base graph. + // Any derived view (window/layer/filter/...) drops it so it recomputes. + schema_cache: Option>, } impl From for GqlGraph { @@ -75,6 +78,21 @@ impl GqlGraph { Self { path, graph: graph.into_dynamic(), + schema_cache: None, + } + } + + /// Carries the base graph's schema cache. Used only for the top-level, unfiltered (e.g. materialized) graph view + /// `cache` is `None` when the view is redacted or type-overridden. + pub fn new_cached( + path: ExistingGraphFolder, + graph: G, + cache: Option>, + ) -> Self { + Self { + path, + graph: graph.into_dynamic(), + schema_cache: cache, } } @@ -86,6 +104,8 @@ impl GqlGraph { Self { path: self.path.clone(), graph: graph_operation(&self.graph).into_dynamic(), + // a derived view's schema differs from the base, so don't cache it + schema_cache: None, } } } @@ -631,7 +651,10 @@ impl GqlGraph { /// Returns the graph schema. async fn schema(&self) -> Result { let self_clone = self.clone(); - Ok(blocking_compute(move || GraphSchema::new(&self_clone.graph)).await) + Ok(blocking_compute(move || { + GraphSchema::new(&self_clone.graph, self_clone.schema_cache.clone()) + }) + .await) } /// Access registered graph algorithms (PageRank, shortest path, etc.) for this diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index f5fc5a17be..1d9e699dfa 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -563,6 +563,7 @@ impl GqlMutableGraph { /// Post mutation operations. async fn post_mutation_ops(&self) { self.graph.set_dirty(true); + self.graph.invalidate_schema_cache(); } } @@ -689,6 +690,7 @@ impl GqlMutableNode { /// Post mutation operations. async fn post_mutation_ops(&self) { self.node.graph.set_dirty(true); + self.node.graph.invalidate_schema_cache(); } } @@ -850,6 +852,7 @@ impl GqlMutableEdge { /// Post mutation operations. async fn post_mutation_ops(&self) { self.edge.graph.set_dirty(true); + self.edge.graph.invalidate_schema_cache(); } } diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index ea5a706c58..1b3d32a28d 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -192,13 +192,13 @@ impl QueryRoot { ) -> Result> { let data = ctx.data_unchecked::(); // Ok(None) = permission denied (hides existence/access level); Err = load failed. - let Some((folder, graph)) = data + let Some((folder, graph, schema_cache)) = data .get_graph_with_read_permission(ctx, path, graph_type) .await? else { return Ok(None); }; - Ok(Some(GqlGraph::new(folder, graph))) + Ok(Some(GqlGraph::new_cached(folder, graph, schema_cache))) } /// Returns lightweight metadata for a graph (node/edge counts, timestamps) without loading it. diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs new file mode 100644 index 0000000000..d06d93c8d4 --- /dev/null +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -0,0 +1,44 @@ +use crate::model::schema::property_schema::PropertySchema; +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; + +/// Cache key for an edge schema entry: `(layer name, src node type, dst node type)`. +type EdgeKey = (String, String, String); + +/// Per-graph cache of computed edge schema results. +/// Lives on the in-memory `GraphWithVectors` and is shared with the base `GqlGraph` view. +#[derive(Default)] +pub(crate) struct SchemaCache { + properties: RwLock>>>, + metadata: RwLock>>>, +} + +impl SchemaCache { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn get_edge_properties(&self, key: &EdgeKey) -> Option>> { + self.properties.read().unwrap().get(key).cloned() + } + + pub(crate) fn store_edge_properties(&self, key: EdgeKey, value: Arc>) { + self.properties.write().unwrap().insert(key, value); + } + + pub(crate) fn get_edge_metadata(&self, key: &EdgeKey) -> Option>> { + self.metadata.read().unwrap().get(key).cloned() + } + + pub(crate) fn store_edge_metadata(&self, key: EdgeKey, value: Arc>) { + self.metadata.write().unwrap().insert(key, value); + } + + /// Drop all cached schema data. Called on any graph mutation. + pub(crate) fn invalidate(&self) { + self.properties.write().unwrap().clear(); + self.metadata.write().unwrap().clear(); + } +} diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index f193650156..22134c4f1c 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -1,6 +1,6 @@ use crate::{ model::schema::{ - property_schema::PropertySchema, SchemaAggregate, ENUM_BOUNDARY, + cache::SchemaCache, property_schema::PropertySchema, SchemaAggregate, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES, }, rayon::blocking_compute, @@ -24,25 +24,45 @@ use std::{ #[derive(Clone, ResolvedObject)] pub(crate) struct EdgeSchema { graph: G, + layer: String, src_type: String, dst_type: String, - // scan once and remember edges matching the (srcType, dstType) + // scan once and remember edges matching the (srcNodeType, dstNodeType) edges: Arc<[EdgeRef]>, + // schema cache for the base graph, `None` for filtered/derived views + cache: Option>, } impl EdgeSchema { - pub fn new(graph: G, src_type: String, dst_type: String, edges: Vec) -> Self { + pub fn new( + graph: G, + layer: String, + src_type: String, + dst_type: String, + edges: Vec, + cache: Option>, + ) -> Self { Self { graph, + layer, src_type, dst_type, edges: edges.into(), + cache, } } fn edges(&self) -> impl Iterator> + '_ { self.edges.iter().map(|e| EdgeView::new(&self.graph, *e)) } + + fn cache_key(&self) -> (String, String, String) { + ( + self.layer.clone(), + self.src_type.clone(), + self.dst_type.clone(), + ) + } } #[ResolvedObjectFields] @@ -59,8 +79,14 @@ impl EdgeSchema { /// Returns the list of property schemas for edges matching these `(src_node_type, dst_node_type)` async fn properties(&self) -> Vec { + if let Some(cache) = &self.cache { + if let Some(hit) = cache.get_edge_properties(&self.cache_key()) { + return hit.as_ref().clone(); + } + } + // not found in cache, so calculate it and cache it let cloned = self.clone(); - blocking_compute(move || { + let result = blocking_compute(move || { if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { // large graph, do not collect detailed schema as it is expensive let visible: HashSet = @@ -84,12 +110,22 @@ impl EdgeSchema { ) } }) - .await + .await; + if let Some(cache) = &self.cache { + cache.store_edge_properties(self.cache_key(), Arc::new(result.clone())); + } + result } /// Returns the list of metadata schemas for edges matching these `(src_node_type, dst_node_type)` async fn metadata(&self) -> Vec { + if let Some(cache) = &self.cache { + if let Some(hit) = cache.get_edge_metadata(&self.cache_key()) { + return hit.as_ref().clone(); + } + } + // not found in cache, so calculate it and cache it let cloned = self.clone(); - blocking_compute(move || { + let result = blocking_compute(move || { if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { // large graph, do not collect detailed schema as it is expensive let visible: HashSet = cloned.graph.edge_visible_metadata_ids().collect(); @@ -112,7 +148,11 @@ impl EdgeSchema { ) } }) - .await + .await; + if let Some(cache) = &self.cache { + cache.store_edge_metadata(self.cache_key(), Arc::new(result.clone())); + } + result } } diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index f0c007ae39..672a2dc4d7 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -1,8 +1,11 @@ -use crate::model::schema::{layer_schema::LayerSchema, node_schema::NodeSchema}; +use crate::model::schema::{ + cache::SchemaCache, layer_schema::LayerSchema, node_schema::NodeSchema, +}; use dynamic_graphql::SimpleObject; use itertools::Itertools; use raphtory::{db::api::view::DynamicGraph, prelude::*}; use raphtory_storage::core_ops::CoreGraphOps; +use std::sync::Arc; #[derive(SimpleObject)] pub(crate) struct GraphSchema { @@ -11,7 +14,9 @@ pub(crate) struct GraphSchema { } impl GraphSchema { - pub fn new(graph: &DynamicGraph) -> Self { + /// `cache` is `Some` only for the unfiltered, base graph. + /// Other views (such as filtered, layered, windowed) pass `None` and recompute every time. + pub fn new(graph: &DynamicGraph, cache: Option>) -> Self { let node_types = graph.node_meta().node_type_meta().ids(); let nodes = node_types .map(|node_type| NodeSchema::new(node_type, graph.clone())) @@ -19,7 +24,7 @@ impl GraphSchema { let layers = graph .unique_layers() - .map(|layer_name| graph.layers(layer_name).unwrap().into()) + .map(|layer_name| LayerSchema::new(graph.layers(layer_name).unwrap(), cache.clone())) .collect_vec(); GraphSchema { nodes, layers } diff --git a/raphtory-graphql/src/model/schema/layer_schema.rs b/raphtory-graphql/src/model/schema/layer_schema.rs index 7ac7a70b26..283058ebb3 100644 --- a/raphtory-graphql/src/model/schema/layer_schema.rs +++ b/raphtory-graphql/src/model/schema/layer_schema.rs @@ -1,5 +1,7 @@ use crate::{ - model::schema::{edge_schema::EdgeSchema, get_node_type, MAX_DETAILED_SCHEMA_ENTITIES}, + model::schema::{ + cache::SchemaCache, edge_schema::EdgeSchema, get_node_type, MAX_DETAILED_SCHEMA_ENTITIES, + }, rayon::blocking_compute, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; @@ -8,18 +10,20 @@ use raphtory::{ prelude::*, }; use raphtory_api::core::entities::edges::edge_ref::EdgeRef; -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; /// Describes a single edge layer — its name and the per `(srcType, dstType)` /// edge schemas observed within it. #[derive(ResolvedObject)] pub(crate) struct LayerSchema { graph: LayeredGraph, + // schema cache for the base graph; `None` for filtered views + cache: Option>, } -impl From> for LayerSchema { - fn from(value: LayeredGraph) -> Self { - Self { graph: value } +impl LayerSchema { + pub fn new(graph: LayeredGraph, cache: Option>) -> Self { + Self { graph, cache } } } @@ -38,7 +42,13 @@ impl LayerSchema { /// Returns the list of edge schemas for this edge layer async fn edges(&self) -> Vec>> { let graph = self.graph.clone(); + let cache = self.cache.clone(); blocking_compute(move || { + let layer: String = graph + .unique_layers() + .next() + .map(Into::into) + .unwrap_or_default(); // Single scan over the layer's edges, bucketing them by (src_node_type, dst_node_type) let mut buckets: HashMap<(String, String), Vec> = HashMap::new(); let mut total = 0usize; @@ -59,7 +69,14 @@ impl LayerSchema { buckets .into_iter() .map(|((src_type, dst_type), edges)| { - EdgeSchema::new(graph.clone(), src_type, dst_type, edges) + EdgeSchema::new( + graph.clone(), + layer.clone(), + src_type, + dst_type, + edges, + cache.clone(), + ) }) .collect() }) diff --git a/raphtory-graphql/src/model/schema/mod.rs b/raphtory-graphql/src/model/schema/mod.rs index b7377e70ed..82151320df 100644 --- a/raphtory-graphql/src/model/schema/mod.rs +++ b/raphtory-graphql/src/model/schema/mod.rs @@ -5,6 +5,7 @@ use raphtory::{ use rustc_hash::FxHashMap; use std::collections::HashSet; +pub(crate) mod cache; pub(crate) mod edge_schema; pub(crate) mod graph_schema; pub(crate) mod layer_schema; diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 2ce583ee2c..a276b06515 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -224,7 +224,7 @@ mod test { } fn check_schema(g: &Graph) { - let gs = GraphSchema::new(&g.clone().into_dynamic()); + let gs = GraphSchema::new(&g.clone().into_dynamic(), None); let mut actual: Vec<(String, Vec, Vec)> = gs .nodes diff --git a/raphtory-graphql/src/model/schema/property_schema.rs b/raphtory-graphql/src/model/schema/property_schema.rs index 21e61bc697..7f5d311730 100644 --- a/raphtory-graphql/src/model/schema/property_schema.rs +++ b/raphtory-graphql/src/model/schema/property_schema.rs @@ -1,6 +1,6 @@ use dynamic_graphql::SimpleObject; -#[derive(SimpleObject, Debug, PartialEq, PartialOrd, Ord, Eq)] +#[derive(SimpleObject, Clone, Debug, PartialEq, PartialOrd, Ord, Eq)] pub(crate) struct PropertySchema { key: String, property_type: String, From 0b2c9cc885c777ad201ad3da39858360471ffb2e Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Fri, 19 Jun 2026 05:52:53 -0400 Subject: [PATCH 16/27] Changed schema cache lookup key to be a struct instead of a (String, String, String) type alias. This allows us to pass references to the key easily. Faster on cache lookup --- raphtory-graphql/src/model/schema/cache.rs | 39 +++++++++++++++---- .../src/model/schema/edge_schema.rs | 34 ++++++---------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs index d06d93c8d4..1f868c1207 100644 --- a/raphtory-graphql/src/model/schema/cache.rs +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -5,14 +5,29 @@ use std::{ }; /// Cache key for an edge schema entry: `(layer name, src node type, dst node type)`. -type EdgeKey = (String, String, String); +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub(crate) struct EdgeSchemaKey { + pub layer: String, + pub src_type: String, + pub dst_type: String, +} + +impl EdgeSchemaKey { + pub(crate) fn new(layer: String, src_type: String, dst_type: String) -> Self { + Self { + layer, + src_type, + dst_type, + } + } +} /// Per-graph cache of computed edge schema results. /// Lives on the in-memory `GraphWithVectors` and is shared with the base `GqlGraph` view. #[derive(Default)] pub(crate) struct SchemaCache { - properties: RwLock>>>, - metadata: RwLock>>>, + properties: RwLock>>>, + metadata: RwLock>>>, } impl SchemaCache { @@ -20,19 +35,29 @@ impl SchemaCache { Self::default() } - pub(crate) fn get_edge_properties(&self, key: &EdgeKey) -> Option>> { + pub(crate) fn get_edge_properties( + &self, + key: &EdgeSchemaKey, + ) -> Option>> { self.properties.read().unwrap().get(key).cloned() } - pub(crate) fn store_edge_properties(&self, key: EdgeKey, value: Arc>) { + pub(crate) fn store_edge_properties( + &self, + key: EdgeSchemaKey, + value: Arc>, + ) { self.properties.write().unwrap().insert(key, value); } - pub(crate) fn get_edge_metadata(&self, key: &EdgeKey) -> Option>> { + pub(crate) fn get_edge_metadata( + &self, + key: &EdgeSchemaKey, + ) -> Option>> { self.metadata.read().unwrap().get(key).cloned() } - pub(crate) fn store_edge_metadata(&self, key: EdgeKey, value: Arc>) { + pub(crate) fn store_edge_metadata(&self, key: EdgeSchemaKey, value: Arc>) { self.metadata.write().unwrap().insert(key, value); } diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 22134c4f1c..efbb34d454 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -1,7 +1,8 @@ use crate::{ model::schema::{ - cache::SchemaCache, property_schema::PropertySchema, SchemaAggregate, ENUM_BOUNDARY, - MAX_DETAILED_SCHEMA_ENTITIES, + cache::{EdgeSchemaKey, SchemaCache}, + property_schema::PropertySchema, + SchemaAggregate, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES, }, rayon::blocking_compute, }; @@ -24,9 +25,8 @@ use std::{ #[derive(Clone, ResolvedObject)] pub(crate) struct EdgeSchema { graph: G, - layer: String, - src_type: String, - dst_type: String, + // (layer, src_type, dst_type) information; also the schema-cache lookup key + key: EdgeSchemaKey, // scan once and remember edges matching the (srcNodeType, dstNodeType) edges: Arc<[EdgeRef]>, // schema cache for the base graph, `None` for filtered/derived views @@ -44,9 +44,7 @@ impl EdgeSchema { ) -> Self { Self { graph, - layer, - src_type, - dst_type, + key: EdgeSchemaKey::new(layer, src_type, dst_type), edges: edges.into(), cache, } @@ -55,32 +53,24 @@ impl EdgeSchema { fn edges(&self) -> impl Iterator> + '_ { self.edges.iter().map(|e| EdgeView::new(&self.graph, *e)) } - - fn cache_key(&self) -> (String, String, String) { - ( - self.layer.clone(), - self.src_type.clone(), - self.dst_type.clone(), - ) - } } #[ResolvedObjectFields] impl EdgeSchema { /// Returns the type of source for these edges async fn src_type(&self) -> String { - self.src_type.clone() + self.key.src_type.clone() } /// Returns the type of destination for these edges async fn dst_type(&self) -> String { - self.dst_type.clone() + self.key.dst_type.clone() } /// Returns the list of property schemas for edges matching these `(src_node_type, dst_node_type)` async fn properties(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.get_edge_properties(&self.cache_key()) { + if let Some(hit) = cache.get_edge_properties(&self.key) { return hit.as_ref().clone(); } } @@ -112,14 +102,14 @@ impl EdgeSchema { }) .await; if let Some(cache) = &self.cache { - cache.store_edge_properties(self.cache_key(), Arc::new(result.clone())); + cache.store_edge_properties(self.key.clone(), Arc::new(result.clone())); } result } /// Returns the list of metadata schemas for edges matching these `(src_node_type, dst_node_type)` async fn metadata(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.get_edge_metadata(&self.cache_key()) { + if let Some(hit) = cache.get_edge_metadata(&self.key) { return hit.as_ref().clone(); } } @@ -150,7 +140,7 @@ impl EdgeSchema { }) .await; if let Some(cache) = &self.cache { - cache.store_edge_metadata(self.cache_key(), Arc::new(result.clone())); + cache.store_edge_metadata(self.key.clone(), Arc::new(result.clone())); } result } From 5abe811b07ac4dd87daa65ba14fc64e2f0991705 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Mon, 22 Jun 2026 04:04:25 -0400 Subject: [PATCH 17/27] Added a node schema cache --- raphtory-graphql/src/model/schema/cache.rs | 94 +++++++++++++++---- .../src/model/schema/edge_schema.rs | 12 ++- .../src/model/schema/graph_schema.rs | 2 +- .../src/model/schema/node_schema.rs | 51 ++++++++-- 4 files changed, 124 insertions(+), 35 deletions(-) diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs index 1f868c1207..23971e36da 100644 --- a/raphtory-graphql/src/model/schema/cache.rs +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -22,12 +22,24 @@ impl EdgeSchemaKey { } } -/// Per-graph cache of computed edge schema results. -/// Lives on the in-memory `GraphWithVectors` and is shared with the base `GqlGraph` view. +/// Cache key for a node schema entry: the node type id. +#[derive(Clone, PartialEq, Eq, Hash, Debug)] +pub(crate) struct NodeSchemaKey { + pub type_id: usize, +} + +impl NodeSchemaKey { + pub(crate) fn new(type_id: usize) -> Self { + Self { type_id } + } +} + +/// Root per-graph schema cache. Lives on the in-memory `GraphWithVectors` and is +/// shared with the base `GqlGraph` view. #[derive(Default)] pub(crate) struct SchemaCache { - properties: RwLock>>>, - metadata: RwLock>>>, + edge: EdgeSchemaCache, + node: NodeSchemaCache, } impl SchemaCache { @@ -35,34 +47,76 @@ impl SchemaCache { Self::default() } - pub(crate) fn get_edge_properties( - &self, - key: &EdgeSchemaKey, - ) -> Option>> { + pub(crate) fn edge(&self) -> &EdgeSchemaCache { + &self.edge + } + + pub(crate) fn node(&self) -> &NodeSchemaCache { + &self.node + } + + /// Drop all cached schema data. Called on any graph mutation. + pub(crate) fn invalidate(&self) { + self.edge.invalidate(); + self.node.invalidate(); + } +} + +/// Cache of computed edge schema results, keyed by `(layer, src_type, dst_type)`. +#[derive(Default)] +pub(crate) struct EdgeSchemaCache { + properties: RwLock>>>, + metadata: RwLock>>>, +} + +impl EdgeSchemaCache { + pub(crate) fn get_properties(&self, key: &EdgeSchemaKey) -> Option>> { + self.properties.read().unwrap().get(key).cloned() + } + + pub(crate) fn store_properties(&self, key: EdgeSchemaKey, value: Arc>) { + self.properties.write().unwrap().insert(key, value); + } + + pub(crate) fn get_metadata(&self, key: &EdgeSchemaKey) -> Option>> { + self.metadata.read().unwrap().get(key).cloned() + } + + pub(crate) fn store_metadata(&self, key: EdgeSchemaKey, value: Arc>) { + self.metadata.write().unwrap().insert(key, value); + } + + fn invalidate(&self) { + self.properties.write().unwrap().clear(); + self.metadata.write().unwrap().clear(); + } +} + +/// Cache of computed node schema results, keyed by node type id. +#[derive(Default)] +pub(crate) struct NodeSchemaCache { + properties: RwLock>>>, + metadata: RwLock>>>, +} + +impl NodeSchemaCache { + pub(crate) fn get_properties(&self, key: &NodeSchemaKey) -> Option>> { self.properties.read().unwrap().get(key).cloned() } - pub(crate) fn store_edge_properties( - &self, - key: EdgeSchemaKey, - value: Arc>, - ) { + pub(crate) fn store_properties(&self, key: NodeSchemaKey, value: Arc>) { self.properties.write().unwrap().insert(key, value); } - pub(crate) fn get_edge_metadata( - &self, - key: &EdgeSchemaKey, - ) -> Option>> { + pub(crate) fn get_metadata(&self, key: &NodeSchemaKey) -> Option>> { self.metadata.read().unwrap().get(key).cloned() } - pub(crate) fn store_edge_metadata(&self, key: EdgeSchemaKey, value: Arc>) { + pub(crate) fn store_metadata(&self, key: NodeSchemaKey, value: Arc>) { self.metadata.write().unwrap().insert(key, value); } - /// Drop all cached schema data. Called on any graph mutation. - pub(crate) fn invalidate(&self) { + fn invalidate(&self) { self.properties.write().unwrap().clear(); self.metadata.write().unwrap().clear(); } diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index efbb34d454..a055c0ad51 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -70,7 +70,7 @@ impl EdgeSchema { /// Returns the list of property schemas for edges matching these `(src_node_type, dst_node_type)` async fn properties(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.get_edge_properties(&self.key) { + if let Some(hit) = cache.edge().get_properties(&self.key) { return hit.as_ref().clone(); } } @@ -102,14 +102,16 @@ impl EdgeSchema { }) .await; if let Some(cache) = &self.cache { - cache.store_edge_properties(self.key.clone(), Arc::new(result.clone())); + cache + .edge() + .store_properties(self.key.clone(), Arc::new(result.clone())); } result } /// Returns the list of metadata schemas for edges matching these `(src_node_type, dst_node_type)` async fn metadata(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.get_edge_metadata(&self.key) { + if let Some(hit) = cache.edge().get_metadata(&self.key) { return hit.as_ref().clone(); } } @@ -140,7 +142,9 @@ impl EdgeSchema { }) .await; if let Some(cache) = &self.cache { - cache.store_edge_metadata(self.key.clone(), Arc::new(result.clone())); + cache + .edge() + .store_metadata(self.key.clone(), Arc::new(result.clone())); } result } diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index 672a2dc4d7..cb550347ef 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -19,7 +19,7 @@ impl GraphSchema { pub fn new(graph: &DynamicGraph, cache: Option>) -> Self { let node_types = graph.node_meta().node_type_meta().ids(); let nodes = node_types - .map(|node_type| NodeSchema::new(node_type, graph.clone())) + .map(|node_type| NodeSchema::new(node_type, graph.clone(), cache.clone())) .collect(); let layers = graph diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index a276b06515..84c96aa4fe 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -1,5 +1,7 @@ use crate::model::schema::{ - property_schema::PropertySchema, DEFAULT_NODE_TYPE, MAX_DETAILED_SCHEMA_ENTITIES, + cache::{NodeSchemaKey, SchemaCache}, + property_schema::PropertySchema, + DEFAULT_NODE_TYPE, MAX_DETAILED_SCHEMA_ENTITIES, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use raphtory::{ @@ -16,21 +18,26 @@ use raphtory::{ use raphtory_api::core::entities::LayerIds; use raphtory_storage::core_ops::CoreGraphOps; use rayon::prelude::*; +use std::sync::Arc; /// Describes nodes of a specific type in a graph — its property keys and /// observed value types (and, for string-valued properties, the set of /// distinct values seen). One `NodeSchema` per node type. -#[derive(ResolvedObject)] +#[derive(Clone, ResolvedObject)] pub(crate) struct NodeSchema { - pub(crate) type_id: usize, + /// node type id; also the schema-cache lookup key + key: NodeSchemaKey, graph: DynamicGraph, + // schema cache for the base graph, `None` for filtered/derived views + cache: Option>, } impl NodeSchema { - pub fn new(node_type: usize, graph: DynamicGraph) -> Self { + pub fn new(node_type: usize, graph: DynamicGraph, cache: Option>) -> Self { Self { - type_id: node_type, + key: NodeSchemaKey::new(node_type), graph, + cache, } } } @@ -47,13 +54,37 @@ impl NodeSchema { /// ever set on a node of this type, with its observed `PropertyType` and (for /// string-valued properties) the set of distinct values. async fn properties(&self) -> Vec { - self.properties_inner() + if let Some(cache) = &self.cache { + if let Some(hit) = cache.node().get_properties(&self.key) { + return hit.as_ref().clone(); + } + } + // not found in cache, so calculate it and cache it + let result = self.properties_inner(); + if let Some(cache) = &self.cache { + cache + .node() + .store_properties(self.key.clone(), Arc::new(result.clone())); + } + result } /// Metadata schemas seen on nodes of this type — like `properties`, but /// covering metadata fields rather than temporal properties. async fn metadata(&self) -> Vec { - self.metadata_inner() + if let Some(cache) = &self.cache { + if let Some(hit) = cache.node().get_metadata(&self.key) { + return hit.as_ref().clone(); + } + } + // not found in cache, so calculate it and cache it + let result = self.metadata_inner(); + if let Some(cache) = &self.cache { + cache + .node() + .store_metadata(self.key.clone(), Arc::new(result.clone())); + } + result } } @@ -61,7 +92,7 @@ impl NodeSchema { fn type_name_inner(&self) -> String { self.graph .node_meta() - .get_node_type_name_by_id(self.type_id) + .get_node_type_name_by_id(self.key.type_id) .map(|type_name| type_name.to_string()) .unwrap_or_else(|| DEFAULT_NODE_TYPE.to_string()) } @@ -90,7 +121,7 @@ impl NodeSchema { .filter_map(|(key, dtype)| { let mut node_types_filter = vec![false; self.graph.node_meta().node_type_meta().num_all_fields()]; - node_types_filter[self.type_id] = true; + node_types_filter[self.key.type_id] = true; let filter = TypeId.mask(node_types_filter.into()); let unique_values: ahash::HashSet<_> = NodeFilteredGraph::new(self.graph.clone(), filter) @@ -140,7 +171,7 @@ impl NodeSchema { .filter_map(|(key, dtype)| { let mut node_types_filter = vec![false; self.graph.node_meta().node_type_meta().num_all_fields()]; - node_types_filter[self.type_id] = true; + node_types_filter[self.key.type_id] = true; let filter = TypeId.mask(node_types_filter.into()); let unique_values: ahash::HashSet<_> = NodeFilteredGraph::new(self.graph.clone(), filter) From 14c89dab0352c0b68b272eb043ebdd610849c2ec Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Mon, 22 Jun 2026 05:54:09 -0400 Subject: [PATCH 18/27] Get rid of Arc on Vec in the cache. --- raphtory-graphql/src/model/schema/cache.rs | 29 +++++++++---------- .../src/model/schema/edge_schema.rs | 8 ++--- .../src/model/schema/node_schema.rs | 8 ++--- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs index 23971e36da..b0e341d8f9 100644 --- a/raphtory-graphql/src/model/schema/cache.rs +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -1,8 +1,5 @@ use crate::model::schema::property_schema::PropertySchema; -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; +use std::{collections::HashMap, sync::RwLock}; /// Cache key for an edge schema entry: `(layer name, src node type, dst node type)`. #[derive(Clone, PartialEq, Eq, Hash, Debug)] @@ -65,24 +62,24 @@ impl SchemaCache { /// Cache of computed edge schema results, keyed by `(layer, src_type, dst_type)`. #[derive(Default)] pub(crate) struct EdgeSchemaCache { - properties: RwLock>>>, - metadata: RwLock>>>, + properties: RwLock>>, + metadata: RwLock>>, } impl EdgeSchemaCache { - pub(crate) fn get_properties(&self, key: &EdgeSchemaKey) -> Option>> { + pub(crate) fn get_properties(&self, key: &EdgeSchemaKey) -> Option> { self.properties.read().unwrap().get(key).cloned() } - pub(crate) fn store_properties(&self, key: EdgeSchemaKey, value: Arc>) { + pub(crate) fn store_properties(&self, key: EdgeSchemaKey, value: Vec) { self.properties.write().unwrap().insert(key, value); } - pub(crate) fn get_metadata(&self, key: &EdgeSchemaKey) -> Option>> { + pub(crate) fn get_metadata(&self, key: &EdgeSchemaKey) -> Option> { self.metadata.read().unwrap().get(key).cloned() } - pub(crate) fn store_metadata(&self, key: EdgeSchemaKey, value: Arc>) { + pub(crate) fn store_metadata(&self, key: EdgeSchemaKey, value: Vec) { self.metadata.write().unwrap().insert(key, value); } @@ -95,24 +92,24 @@ impl EdgeSchemaCache { /// Cache of computed node schema results, keyed by node type id. #[derive(Default)] pub(crate) struct NodeSchemaCache { - properties: RwLock>>>, - metadata: RwLock>>>, + properties: RwLock>>, + metadata: RwLock>>, } impl NodeSchemaCache { - pub(crate) fn get_properties(&self, key: &NodeSchemaKey) -> Option>> { + pub(crate) fn get_properties(&self, key: &NodeSchemaKey) -> Option> { self.properties.read().unwrap().get(key).cloned() } - pub(crate) fn store_properties(&self, key: NodeSchemaKey, value: Arc>) { + pub(crate) fn store_properties(&self, key: NodeSchemaKey, value: Vec) { self.properties.write().unwrap().insert(key, value); } - pub(crate) fn get_metadata(&self, key: &NodeSchemaKey) -> Option>> { + pub(crate) fn get_metadata(&self, key: &NodeSchemaKey) -> Option> { self.metadata.read().unwrap().get(key).cloned() } - pub(crate) fn store_metadata(&self, key: NodeSchemaKey, value: Arc>) { + pub(crate) fn store_metadata(&self, key: NodeSchemaKey, value: Vec) { self.metadata.write().unwrap().insert(key, value); } diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index a055c0ad51..9f361c2c8c 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -71,7 +71,7 @@ impl EdgeSchema { async fn properties(&self) -> Vec { if let Some(cache) = &self.cache { if let Some(hit) = cache.edge().get_properties(&self.key) { - return hit.as_ref().clone(); + return hit; } } // not found in cache, so calculate it and cache it @@ -104,7 +104,7 @@ impl EdgeSchema { if let Some(cache) = &self.cache { cache .edge() - .store_properties(self.key.clone(), Arc::new(result.clone())); + .store_properties(self.key.clone(), result.clone()); } result } @@ -112,7 +112,7 @@ impl EdgeSchema { async fn metadata(&self) -> Vec { if let Some(cache) = &self.cache { if let Some(hit) = cache.edge().get_metadata(&self.key) { - return hit.as_ref().clone(); + return hit; } } // not found in cache, so calculate it and cache it @@ -144,7 +144,7 @@ impl EdgeSchema { if let Some(cache) = &self.cache { cache .edge() - .store_metadata(self.key.clone(), Arc::new(result.clone())); + .store_metadata(self.key.clone(), result.clone()); } result } diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 84c96aa4fe..8fde5a57c1 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -56,7 +56,7 @@ impl NodeSchema { async fn properties(&self) -> Vec { if let Some(cache) = &self.cache { if let Some(hit) = cache.node().get_properties(&self.key) { - return hit.as_ref().clone(); + return hit; } } // not found in cache, so calculate it and cache it @@ -64,7 +64,7 @@ impl NodeSchema { if let Some(cache) = &self.cache { cache .node() - .store_properties(self.key.clone(), Arc::new(result.clone())); + .store_properties(self.key.clone(), result.clone()); } result } @@ -74,7 +74,7 @@ impl NodeSchema { async fn metadata(&self) -> Vec { if let Some(cache) = &self.cache { if let Some(hit) = cache.node().get_metadata(&self.key) { - return hit.as_ref().clone(); + return hit; } } // not found in cache, so calculate it and cache it @@ -82,7 +82,7 @@ impl NodeSchema { if let Some(cache) = &self.cache { cache .node() - .store_metadata(self.key.clone(), Arc::new(result.clone())); + .store_metadata(self.key.clone(), result.clone()); } result } From 48c243b64b3dbac25d68fe88d85dc9dcbceef85c Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 23 Jun 2026 05:55:04 -0400 Subject: [PATCH 19/27] Clean up comments --- db4-storage/src/pages/edge_page/writer.rs | 2 +- db4-storage/src/pages/node_page/writer.rs | 4 ++-- raphtory-api/src/core/entities/properties/meta.rs | 2 +- raphtory-graphql/src/model/graph/graph.rs | 2 +- raphtory-graphql/src/model/schema/edge_schema.rs | 2 +- raphtory-storage/src/graph/edges/edges.rs | 3 ++- raphtory/src/db/api/properties/internal.rs | 4 ---- .../src/db/api/storage/graph/storage_ops/property_schema.rs | 4 ---- 8 files changed, 8 insertions(+), 15 deletions(-) diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index ae4684abba..6b9b1e7664 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -140,7 +140,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen } // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed + // `.inspect` runs once per emitted item as the iterator is consumed in `insert_edge_internal` let meta = self.writer.edge_meta().clone(); let t_meta = meta.as_ref(); let t_props = t_props.into_iter().inspect(move |(id, _)| { diff --git a/db4-storage/src/pages/node_page/writer.rs b/db4-storage/src/pages/node_page/writer.rs index cfc5692025..4cee0e6b30 100644 --- a/db4-storage/src/pages/node_page/writer.rs +++ b/db4-storage/src/pages/node_page/writer.rs @@ -156,7 +156,7 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri ) { self.l_counter.update_time(t.t()); // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed + // `.inspect` runs once per emitted item as the iterator is consumed in add_props let meta = self.mut_segment.node_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.temporal_prop_mapper() @@ -186,7 +186,7 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri props: impl IntoIterator, ) { // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed + // `.inspect` runs once per emitted item as the iterator is consumed in update_metadata let meta = self.mut_segment.node_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); diff --git a/raphtory-api/src/core/entities/properties/meta.rs b/raphtory-api/src/core/entities/properties/meta.rs index 9f2ffcfc9d..9dba56934d 100644 --- a/raphtory-api/src/core/entities/properties/meta.rs +++ b/raphtory-api/src/core/entities/properties/meta.rs @@ -261,7 +261,7 @@ pub struct PropMapper { /// Estimated size in bytes of a single row of properties maintained by this mapper. row_size: AtomicUsize, - /// Per-layer property presence bitset; `layer_presence[layer_id][prop_id]` + /// Per-layer property presence bitset; `layer_prop_presence[layer_id][prop_id]` /// is true iff this property has been observed in this layer layer_prop_presence: Arc>>>, } diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index 5921ff8217..c736ddb968 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -104,7 +104,7 @@ impl GqlGraph { Self { path: self.path.clone(), graph: graph_operation(&self.graph).into_dynamic(), - // a derived view's schema differs from the base, so don't cache it + // a derived view's schema differs from the base, so don't use the cache schema_cache: None, } } diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs index 9f361c2c8c..f5d1f23f21 100644 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ b/raphtory-graphql/src/model/schema/edge_schema.rs @@ -27,7 +27,7 @@ pub(crate) struct EdgeSchema { graph: G, // (layer, src_type, dst_type) information; also the schema-cache lookup key key: EdgeSchemaKey, - // scan once and remember edges matching the (srcNodeType, dstNodeType) + // already scanned the edges matching the (srcNodeType, dstNodeType) in LayerSchema edges: Arc<[EdgeRef]>, // schema cache for the base graph, `None` for filtered/derived views cache: Option>, diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 64c27c15f6..9689b8c14d 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -149,7 +149,7 @@ impl<'a> EdgesStorageRef<'a> { } /// O(1) check via the per-layer bitset cached on `Meta.temporal_prop_mapper`. - /// `false` is authoritative — callers can skip column reads for `(layer_id, prop_id)`. + /// `false` is authoritative i.e. callers can skip column reads for `(layer_id, prop_id)`. pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { let inner = match self { EdgesStorageRef::Mem(storage) => storage.storage(), @@ -162,6 +162,7 @@ impl<'a> EdgesStorageRef<'a> { } /// O(1) check via the per-layer bitset cached on `Meta.metadata_mapper`. + /// `false` is authoritative i.e. callers can skip column reads for `(layer_id, prop_id)`. pub fn layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { let inner = match self { EdgesStorageRef::Mem(storage) => storage.storage(), diff --git a/raphtory/src/db/api/properties/internal.rs b/raphtory/src/db/api/properties/internal.rs index 02e7b221eb..b763d71fa5 100644 --- a/raphtory/src/db/api/properties/internal.rs +++ b/raphtory/src/db/api/properties/internal.rs @@ -23,10 +23,8 @@ pub trait NodePropertySchemaOps: Send + Sync { fn node_visible_metadata_id(&self, name: &str) -> Option; /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn node_visible_metadata_name(&self, id: usize) -> Option; - /// O(1) check: is temporal-prop `prop_id` present on any node in `layer_id`? fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; - /// O(1) check: is metadata-prop `prop_id` present on any node in `layer_id`? fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool; } @@ -41,10 +39,8 @@ pub trait EdgePropertySchemaOps: Send + Sync { fn edge_visible_metadata_id(&self, name: &str) -> Option; /// Returns `None` if `id` is not visible in this view (e.g. redacted). fn edge_visible_metadata_name(&self, id: usize) -> Option; - /// O(1) check: is temporal-prop `prop_id` present on any edge in `layer_id`? fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool; - /// O(1) check: is metadata-prop `prop_id` present on any edge in `layer_id`? fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool; } diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index cc8697fd49..3919f8cd75 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -30,11 +30,9 @@ impl NodePropertySchemaOps for GraphStorage { fn node_visible_metadata_name(&self, id: usize) -> Option { Some(self.node_meta().metadata_mapper().get_name(id).clone()) } - fn node_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { self.nodes().layer_has_temporal_prop(layer_id, prop_id) } - fn node_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { self.nodes().layer_has_metadata(layer_id, prop_id) } @@ -62,11 +60,9 @@ impl EdgePropertySchemaOps for GraphStorage { fn edge_visible_metadata_name(&self, id: usize) -> Option { Some(self.edge_meta().metadata_mapper().get_name(id).clone()) } - fn edge_layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { self.edges().layer_has_temporal_prop(layer_id, prop_id) } - fn edge_layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { self.edges().layer_has_metadata(layer_id, prop_id) } From c69df94455a2cc9be34841d0435245333acd0d49 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 30 Jun 2026 16:31:01 -0400 Subject: [PATCH 20/27] Get rid of EdgeSchema. Instead, we use LayerSchema to report properties and metadata per layer. We use the maintained property presence bitsets to do this --- raphtory-graphql/src/lib.rs | 24 +-- raphtory-graphql/src/model/graph/graph.rs | 2 +- raphtory-graphql/src/model/schema/cache.rs | 54 ----- .../src/model/schema/edge_schema.rs | 188 ------------------ .../src/model/schema/graph_schema.rs | 7 +- .../src/model/schema/layer_schema.rs | 169 ++++++++++------ raphtory-graphql/src/model/schema/mod.rs | 26 +-- 7 files changed, 128 insertions(+), 342 deletions(-) delete mode 100644 raphtory-graphql/src/model/schema/edge_schema.rs diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 1a3f2acfde..3e42d37c1a 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -388,17 +388,15 @@ mod graphql_test { graph(path: "graph") { schema { layers { - edges { - properties { - key - propertyType - variants - } - metadata { - key - propertyType - variants - } + properties { + key + propertyType + variants + } + metadata { + key + propertyType + variants } } nodes { @@ -449,7 +447,7 @@ mod graphql_test { } if let Value::Array(mut edge_properties) = - data["graph"]["schema"]["layers"][0]["edges"][0]["properties"].clone() + data["graph"]["schema"]["layers"][0]["properties"].clone() { sort_properties(&mut edge_properties); @@ -459,7 +457,7 @@ mod graphql_test { } if let Value::Array(mut edge_metadata) = - data["graph"]["schema"]["layers"][0]["edges"][0]["metadata"].clone() + data["graph"]["schema"]["layers"][0]["metadata"].clone() { sort_properties(&mut edge_metadata); diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index 7be037d92b..0957ee8fa4 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -85,7 +85,7 @@ impl GqlGraph { /// Carries the base graph's schema cache. Used only for the top-level, unfiltered (e.g. materialized) graph view /// `cache` is `None` when the view is redacted or type-overridden. pub fn new_cached( - path: ExistingGraphFolder, + path: UnlockedGraphFolder, graph: G, cache: Option>, ) -> Self { diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs index b0e341d8f9..dd87f5036a 100644 --- a/raphtory-graphql/src/model/schema/cache.rs +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -1,24 +1,6 @@ use crate::model::schema::property_schema::PropertySchema; use std::{collections::HashMap, sync::RwLock}; -/// Cache key for an edge schema entry: `(layer name, src node type, dst node type)`. -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub(crate) struct EdgeSchemaKey { - pub layer: String, - pub src_type: String, - pub dst_type: String, -} - -impl EdgeSchemaKey { - pub(crate) fn new(layer: String, src_type: String, dst_type: String) -> Self { - Self { - layer, - src_type, - dst_type, - } - } -} - /// Cache key for a node schema entry: the node type id. #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub(crate) struct NodeSchemaKey { @@ -35,7 +17,6 @@ impl NodeSchemaKey { /// shared with the base `GqlGraph` view. #[derive(Default)] pub(crate) struct SchemaCache { - edge: EdgeSchemaCache, node: NodeSchemaCache, } @@ -44,51 +25,16 @@ impl SchemaCache { Self::default() } - pub(crate) fn edge(&self) -> &EdgeSchemaCache { - &self.edge - } - pub(crate) fn node(&self) -> &NodeSchemaCache { &self.node } /// Drop all cached schema data. Called on any graph mutation. pub(crate) fn invalidate(&self) { - self.edge.invalidate(); self.node.invalidate(); } } -/// Cache of computed edge schema results, keyed by `(layer, src_type, dst_type)`. -#[derive(Default)] -pub(crate) struct EdgeSchemaCache { - properties: RwLock>>, - metadata: RwLock>>, -} - -impl EdgeSchemaCache { - pub(crate) fn get_properties(&self, key: &EdgeSchemaKey) -> Option> { - self.properties.read().unwrap().get(key).cloned() - } - - pub(crate) fn store_properties(&self, key: EdgeSchemaKey, value: Vec) { - self.properties.write().unwrap().insert(key, value); - } - - pub(crate) fn get_metadata(&self, key: &EdgeSchemaKey) -> Option> { - self.metadata.read().unwrap().get(key).cloned() - } - - pub(crate) fn store_metadata(&self, key: EdgeSchemaKey, value: Vec) { - self.metadata.write().unwrap().insert(key, value); - } - - fn invalidate(&self) { - self.properties.write().unwrap().clear(); - self.metadata.write().unwrap().clear(); - } -} - /// Cache of computed node schema results, keyed by node type id. #[derive(Default)] pub(crate) struct NodeSchemaCache { diff --git a/raphtory-graphql/src/model/schema/edge_schema.rs b/raphtory-graphql/src/model/schema/edge_schema.rs deleted file mode 100644 index f5d1f23f21..0000000000 --- a/raphtory-graphql/src/model/schema/edge_schema.rs +++ /dev/null @@ -1,188 +0,0 @@ -use crate::{ - model::schema::{ - cache::{EdgeSchemaKey, SchemaCache}, - property_schema::PropertySchema, - SchemaAggregate, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES, - }, - rayon::blocking_compute, -}; -use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; -use raphtory::{ - db::{api::view::StaticGraphViewOps, graph::edge::EdgeView}, - prelude::*, -}; -use raphtory_api::core::entities::{ - edges::edge_ref::EdgeRef, properties::meta::PropMapper, LayerIds, -}; -use std::{ - collections::{hash_map::Entry, HashSet}, - sync::Arc, -}; - -/// Describes edges between a specific pair of node types — the property and -/// metadata keys seen on such edges, along with their observed value types. -/// One `EdgeSchema` per `(srcType, dstType)` pair per layer. -#[derive(Clone, ResolvedObject)] -pub(crate) struct EdgeSchema { - graph: G, - // (layer, src_type, dst_type) information; also the schema-cache lookup key - key: EdgeSchemaKey, - // already scanned the edges matching the (srcNodeType, dstNodeType) in LayerSchema - edges: Arc<[EdgeRef]>, - // schema cache for the base graph, `None` for filtered/derived views - cache: Option>, -} - -impl EdgeSchema { - pub fn new( - graph: G, - layer: String, - src_type: String, - dst_type: String, - edges: Vec, - cache: Option>, - ) -> Self { - Self { - graph, - key: EdgeSchemaKey::new(layer, src_type, dst_type), - edges: edges.into(), - cache, - } - } - - fn edges(&self) -> impl Iterator> + '_ { - self.edges.iter().map(|e| EdgeView::new(&self.graph, *e)) - } -} - -#[ResolvedObjectFields] -impl EdgeSchema { - /// Returns the type of source for these edges - async fn src_type(&self) -> String { - self.key.src_type.clone() - } - - /// Returns the type of destination for these edges - async fn dst_type(&self) -> String { - self.key.dst_type.clone() - } - - /// Returns the list of property schemas for edges matching these `(src_node_type, dst_node_type)` - async fn properties(&self) -> Vec { - if let Some(cache) = &self.cache { - if let Some(hit) = cache.edge().get_properties(&self.key) { - return hit; - } - } - // not found in cache, so calculate it and cache it - let cloned = self.clone(); - let result = blocking_compute(move || { - if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { - // large graph, do not collect detailed schema as it is expensive - let visible: HashSet = - cloned.graph.edge_visible_temporal_prop_ids().collect(); - cloned - .graph - .edge_meta() - .temporal_prop_mapper() - .locked() - .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id)) - .map(|(_, name, dtype)| { - PropertySchema::new(name.to_string(), dtype.to_string(), vec![]) - }) - .collect() - } else { - let meta = cloned.graph.edge_meta(); - collect_schema( - cloned.edges().map(|edge| edge.properties()), - meta.temporal_prop_mapper(), - ) - } - }) - .await; - if let Some(cache) = &self.cache { - cache - .edge() - .store_properties(self.key.clone(), result.clone()); - } - result - } - /// Returns the list of metadata schemas for edges matching these `(src_node_type, dst_node_type)` - async fn metadata(&self) -> Vec { - if let Some(cache) = &self.cache { - if let Some(hit) = cache.edge().get_metadata(&self.key) { - return hit; - } - } - // not found in cache, so calculate it and cache it - let cloned = self.clone(); - let result = blocking_compute(move || { - if cloned.graph.unfiltered_num_edges(&LayerIds::All) > MAX_DETAILED_SCHEMA_ENTITIES { - // large graph, do not collect detailed schema as it is expensive - let visible: HashSet = cloned.graph.edge_visible_metadata_ids().collect(); - cloned - .graph - .edge_meta() - .metadata_mapper() - .locked() - .iter_ids_and_types() - .filter(|(id, _, _)| visible.contains(id)) - .map(|(_, name, dtype)| { - PropertySchema::new(name.to_string(), dtype.to_string(), vec![]) - }) - .collect() - } else { - let meta = cloned.graph.edge_meta(); - collect_schema( - cloned.edges().map(|edge| edge.metadata()), - meta.metadata_mapper(), - ) - } - }) - .await; - if let Some(cache) = &self.cache { - cache - .edge() - .store_metadata(self.key.clone(), result.clone()); - } - result - } -} - -/// Aggregate `(key, dtype) -> distinct values` across all edges, capping each value set at `ENUM_BOUNDARY` -fn collect_schema( - props_per_edge: impl Iterator, - mapper: &PropMapper, -) -> Vec { - let mut schema = SchemaAggregate::default(); - for props in props_per_edge { - for ((key, value), id) in props.iter().zip(props.ids()) { - let Some(value) = value else { continue }; - let key_with_prop_type = ( - key.to_string(), - mapper - .get_dtype(id) - .expect("type for internal id should always exist") - .to_string(), - ); - match schema.entry(key_with_prop_type) { - Entry::Vacant(entry) => { - entry.insert(HashSet::from([value.to_string()])); - } - Entry::Occupied(mut entry) => { - let variants = entry.get_mut(); - // An empty set means "too many variants" so we skip. - // Otherwise, there should always be at least 1 value in the set. - if !variants.is_empty() { - variants.insert(value.to_string()); - if variants.len() > ENUM_BOUNDARY { - variants.clear(); - } - } - } - } - } - } - schema.into_iter().map(|prop| prop.into()).collect() -} diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index cb550347ef..500ff64957 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -4,7 +4,7 @@ use crate::model::schema::{ use dynamic_graphql::SimpleObject; use itertools::Itertools; use raphtory::{db::api::view::DynamicGraph, prelude::*}; -use raphtory_storage::core_ops::CoreGraphOps; +use raphtory_storage::{core_ops::CoreGraphOps, layer_ops::InternalLayerOps}; use std::sync::Arc; #[derive(SimpleObject)] @@ -23,8 +23,9 @@ impl GraphSchema { .collect(); let layers = graph - .unique_layers() - .map(|layer_name| LayerSchema::new(graph.layers(layer_name).unwrap(), cache.clone())) + .layer_ids() + .iter(graph.num_layers()) + .map(|layer_id| LayerSchema::new(graph.clone(), layer_id)) .collect_vec(); GraphSchema { nodes, layers } diff --git a/raphtory-graphql/src/model/schema/layer_schema.rs b/raphtory-graphql/src/model/schema/layer_schema.rs index 283058ebb3..24486a6f01 100644 --- a/raphtory-graphql/src/model/schema/layer_schema.rs +++ b/raphtory-graphql/src/model/schema/layer_schema.rs @@ -1,29 +1,23 @@ use crate::{ - model::schema::{ - cache::SchemaCache, edge_schema::EdgeSchema, get_node_type, MAX_DETAILED_SCHEMA_ENTITIES, - }, + model::schema::{property_schema::PropertySchema, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES}, rayon::blocking_compute, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; -use raphtory::{ - db::{api::view::StaticGraphViewOps, graph::views::layer_graph::LayeredGraph}, - prelude::*, -}; -use raphtory_api::core::entities::edges::edge_ref::EdgeRef; -use std::{collections::HashMap, sync::Arc}; +use raphtory::{db::api::view::StaticGraphViewOps, prelude::*}; +use raphtory_api::core::entities::{properties::meta::PropMapper, LayerId, LayerIds}; +use std::collections::{hash_map::Entry, HashMap, HashSet}; -/// Describes a single edge layer — its name and the per `(srcType, dstType)` -/// edge schemas observed within it. -#[derive(ResolvedObject)] +/// Describes a single edge layer: its name and the edge property/metadata keys +/// present in it, with observed variants (property values) on small graphs +#[derive(Clone, ResolvedObject)] pub(crate) struct LayerSchema { - graph: LayeredGraph, - // schema cache for the base graph; `None` for filtered views - cache: Option>, + graph: G, + layer_id: LayerId, } impl LayerSchema { - pub fn new(graph: LayeredGraph, cache: Option>) -> Self { - Self { graph, cache } + pub fn new(graph: G, layer_id: LayerId) -> Self { + Self { graph, layer_id } } } @@ -31,55 +25,108 @@ impl LayerSchema { impl LayerSchema { /// Returns the name of the layer with this schema async fn name(&self) -> String { - let mut layers = self.graph.unique_layers(); - let layer = layers.next().expect("Layered graph has a layer"); - debug_assert!( - layers.next().is_none(), - "Layered graph outputted more than one layer name" - ); - layer.into() + self.graph.get_layer_name(self.layer_id).to_string() + } + + /// Returns the list of property schemas present on edges in this layer + async fn properties(&self) -> Vec { + let graph = self.graph.clone(); + let layer_id = self.layer_id; + blocking_compute(move || { + if too_many_edges(&graph, layer_id) { + // too many edges to collect values: keys/types only, no variants + return collect_layer_schema(&graph, layer_id, false); + } + let layer = graph.get_layer_name(layer_id); + let layered = graph.valid_layers(layer); + let mapper = graph.edge_meta().temporal_prop_mapper(); + collect_variants(layered.edges().into_iter().map(|e| e.properties()), mapper) + }) + .await } - /// Returns the list of edge schemas for this edge layer - async fn edges(&self) -> Vec>> { + + /// Returns the list of metadata schemas present on edges in this layer + async fn metadata(&self) -> Vec { let graph = self.graph.clone(); - let cache = self.cache.clone(); + let layer_id = self.layer_id; blocking_compute(move || { - let layer: String = graph - .unique_layers() - .next() - .map(Into::into) - .unwrap_or_default(); - // Single scan over the layer's edges, bucketing them by (src_node_type, dst_node_type) - let mut buckets: HashMap<(String, String), Vec> = HashMap::new(); - let mut total = 0usize; - for edge in graph.edges().into_iter() { - // FIXME: Do we stop if we have over 1000 edges or no? - // total += 1; - // if total > MAX_DETAILED_SCHEMA_ENTITIES { - // // Too many edges to build a detailed schema; abort - // return vec![]; - // } - let src_type = get_node_type(edge.src()); - let dst_type = get_node_type(edge.dst()); - buckets - .entry((src_type, dst_type)) - .or_default() - .push(edge.edge); + if too_many_edges(&graph, layer_id) { + return collect_layer_schema(&graph, layer_id, true); } - buckets - .into_iter() - .map(|((src_type, dst_type), edges)| { - EdgeSchema::new( - graph.clone(), - layer.clone(), - src_type, - dst_type, - edges, - cache.clone(), - ) - }) - .collect() + let layer = graph.get_layer_name(layer_id); + let layered = graph.valid_layers(layer); + let mapper = graph.edge_meta().metadata_mapper(); + collect_variants(layered.edges().into_iter().map(|e| e.metadata()), mapper) }) .await } } + +/// True if the layer holds more edges than we're willing to scan for values. +fn too_many_edges(graph: &G, layer_id: LayerId) -> bool { + graph.unfiltered_num_edges(&LayerIds::One(layer_id)) > MAX_DETAILED_SCHEMA_ENTITIES +} + +/// Get edge property/metadata keys and types using bitset without collecting values. +/// Redacted properties are handled by the GraphView (in edge_layer_has_*). +fn collect_layer_schema( + graph: &G, + layer_id: LayerId, + metadata: bool, +) -> Vec { + let meta = graph.edge_meta(); + let mapper = if metadata { + meta.metadata_mapper() + } else { + meta.temporal_prop_mapper() + }; + mapper + .locked() + .iter_ids_and_types() + .filter(|(id, _, _)| { + if metadata { + graph.edge_layer_has_metadata(layer_id, *id) + } else { + graph.edge_layer_has_temporal_prop(layer_id, *id) + } + }) + .map(|(_, name, dtype)| PropertySchema::new(name.to_string(), dtype.to_string(), vec![])) + .collect() +} + +/// Collect distinct property values for `(key, dtype)` pairs. If there are too many values, we stop. +fn collect_variants( + props_per_edge: impl Iterator, + mapper: &PropMapper, +) -> Vec { + let mut schema: HashMap<(String, String), HashSet> = HashMap::new(); + for props in props_per_edge { + for ((key, value), id) in props.iter().zip(props.ids()) { + let Some(value) = value else { continue }; + let key_with_prop_type = ( + key.to_string(), + mapper + .get_dtype(id) + .expect("type for internal id should always exist") + .to_string(), + ); + match schema.entry(key_with_prop_type) { + Entry::Vacant(entry) => { + entry.insert(HashSet::from([value.to_string()])); + } + Entry::Occupied(mut entry) => { + let variants = entry.get_mut(); + // An empty set means "too many variants", so we skip + // Otherwise, there should always be at least 1 value in the set + if !variants.is_empty() { + variants.insert(value.to_string()); + if variants.len() > ENUM_BOUNDARY { + variants.clear(); + } + } + } + } + } + } + schema.into_iter().map(|prop| prop.into()).collect() +} diff --git a/raphtory-graphql/src/model/schema/mod.rs b/raphtory-graphql/src/model/schema/mod.rs index 82151320df..4910769977 100644 --- a/raphtory-graphql/src/model/schema/mod.rs +++ b/raphtory-graphql/src/model/schema/mod.rs @@ -1,33 +1,15 @@ -use raphtory::{ - db::graph::node::NodeView, - prelude::{GraphViewOps, NodeViewOps}, -}; -use rustc_hash::FxHashMap; -use std::collections::HashSet; - pub(crate) mod cache; -pub(crate) mod edge_schema; pub(crate) mod graph_schema; pub(crate) mod layer_schema; pub(crate) mod node_schema; pub(crate) mod property_schema; +/// Maximum number of distinct values collected per property key. More than that and we don't report any. const ENUM_BOUNDARY: usize = 20; -/// Above this many entities (nodes graph-wide for `NodeSchema`, edges -/// graph-wide for `EdgeSchema`), schema resolvers skip collecting property -/// values (variants) and only return keys and types +/// Above this many entities (nodes graph-wide for `NodeSchema`, edges in the +/// layer for `LayerSchema`), schema resolvers skip collecting property values +/// (variants) and only return keys and types const MAX_DETAILED_SCHEMA_ENTITIES: usize = 1000; const DEFAULT_NODE_TYPE: &'static str = "None"; - -fn get_node_type<'graph, G: GraphViewOps<'graph>>(node: NodeView<'graph, G>) -> String { - match node.node_type() { - None => "None".into(), - Some(n) => n.to_string(), - } -} - -/// Maps each `(property key, property type)` to its distinct values. -/// An empty HashSet means "too many values" so we're skipping value collection. -type SchemaAggregate = FxHashMap<(String, String), HashSet>; From 5a5a314f32a1cf65b6ce0e4d551db43a263875c7 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 30 Jun 2026 19:20:26 -0400 Subject: [PATCH 21/27] Re-introduce the cache that was for EdgeSchema (now LayerSchema). Get rid of NodeSchemaKey. Wire the cache through in the schemas. --- raphtory-graphql/src/model/schema/cache.rs | 63 +++++++++++++------ .../src/model/schema/graph_schema.rs | 5 +- .../src/model/schema/layer_schema.rs | 48 +++++++++++--- .../src/model/schema/node_schema.rs | 27 ++++---- 4 files changed, 100 insertions(+), 43 deletions(-) diff --git a/raphtory-graphql/src/model/schema/cache.rs b/raphtory-graphql/src/model/schema/cache.rs index dd87f5036a..08218ed3b2 100644 --- a/raphtory-graphql/src/model/schema/cache.rs +++ b/raphtory-graphql/src/model/schema/cache.rs @@ -1,23 +1,13 @@ use crate::model::schema::property_schema::PropertySchema; +use raphtory_api::core::entities::LayerId; use std::{collections::HashMap, sync::RwLock}; -/// Cache key for a node schema entry: the node type id. -#[derive(Clone, PartialEq, Eq, Hash, Debug)] -pub(crate) struct NodeSchemaKey { - pub type_id: usize, -} - -impl NodeSchemaKey { - pub(crate) fn new(type_id: usize) -> Self { - Self { type_id } - } -} - /// Root per-graph schema cache. Lives on the in-memory `GraphWithVectors` and is /// shared with the base `GqlGraph` view. #[derive(Default)] pub(crate) struct SchemaCache { node: NodeSchemaCache, + layer: LayerSchemaCache, } impl SchemaCache { @@ -29,33 +19,70 @@ impl SchemaCache { &self.node } + pub(crate) fn layer(&self) -> &LayerSchemaCache { + &self.layer + } + /// Drop all cached schema data. Called on any graph mutation. pub(crate) fn invalidate(&self) { self.node.invalidate(); + self.layer.invalidate(); } } /// Cache of computed node schema results, keyed by node type id. #[derive(Default)] pub(crate) struct NodeSchemaCache { - properties: RwLock>>, - metadata: RwLock>>, + properties: RwLock>>, + metadata: RwLock>>, } impl NodeSchemaCache { - pub(crate) fn get_properties(&self, key: &NodeSchemaKey) -> Option> { + pub(crate) fn get_properties(&self, type_id: usize) -> Option> { + self.properties.read().unwrap().get(&type_id).cloned() + } + + pub(crate) fn store_properties(&self, type_id: usize, value: Vec) { + self.properties.write().unwrap().insert(type_id, value); + } + + pub(crate) fn get_metadata(&self, type_id: usize) -> Option> { + self.metadata.read().unwrap().get(&type_id).cloned() + } + + pub(crate) fn store_metadata(&self, type_id: usize, value: Vec) { + self.metadata.write().unwrap().insert(type_id, value); + } + + fn invalidate(&self) { + self.properties.write().unwrap().clear(); + self.metadata.write().unwrap().clear(); + } +} + +/// Cache of computed layer schema results, keyed by layer id. Caches the +/// resolved property/metadata lists for a layer whether or not variants were +/// collected, so a repeated `schema` query reuses them until the next mutation. +#[derive(Default)] +pub(crate) struct LayerSchemaCache { + properties: RwLock>>, + metadata: RwLock>>, +} + +impl LayerSchemaCache { + pub(crate) fn get_properties(&self, key: &LayerId) -> Option> { self.properties.read().unwrap().get(key).cloned() } - pub(crate) fn store_properties(&self, key: NodeSchemaKey, value: Vec) { + pub(crate) fn store_properties(&self, key: LayerId, value: Vec) { self.properties.write().unwrap().insert(key, value); } - pub(crate) fn get_metadata(&self, key: &NodeSchemaKey) -> Option> { + pub(crate) fn get_metadata(&self, key: &LayerId) -> Option> { self.metadata.read().unwrap().get(key).cloned() } - pub(crate) fn store_metadata(&self, key: NodeSchemaKey, value: Vec) { + pub(crate) fn store_metadata(&self, key: LayerId, value: Vec) { self.metadata.write().unwrap().insert(key, value); } diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index 500ff64957..83d6e6fe72 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -4,6 +4,7 @@ use crate::model::schema::{ use dynamic_graphql::SimpleObject; use itertools::Itertools; use raphtory::{db::api::view::DynamicGraph, prelude::*}; +use raphtory_api::core::entities::properties::meta::STATIC_GRAPH_LAYER_ID; use raphtory_storage::{core_ops::CoreGraphOps, layer_ops::InternalLayerOps}; use std::sync::Arc; @@ -25,7 +26,9 @@ impl GraphSchema { let layers = graph .layer_ids() .iter(graph.num_layers()) - .map(|layer_id| LayerSchema::new(graph.clone(), layer_id)) + // skip the internal `_static_graph` layer + .filter(|&layer_id| layer_id != STATIC_GRAPH_LAYER_ID) + .map(|layer_id| LayerSchema::new(graph.clone(), layer_id, cache.clone())) .collect_vec(); GraphSchema { nodes, layers } diff --git a/raphtory-graphql/src/model/schema/layer_schema.rs b/raphtory-graphql/src/model/schema/layer_schema.rs index 24486a6f01..8696beb3dc 100644 --- a/raphtory-graphql/src/model/schema/layer_schema.rs +++ b/raphtory-graphql/src/model/schema/layer_schema.rs @@ -1,11 +1,17 @@ use crate::{ - model::schema::{property_schema::PropertySchema, ENUM_BOUNDARY, MAX_DETAILED_SCHEMA_ENTITIES}, + model::schema::{ + cache::SchemaCache, property_schema::PropertySchema, ENUM_BOUNDARY, + MAX_DETAILED_SCHEMA_ENTITIES, + }, rayon::blocking_compute, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use raphtory::{db::api::view::StaticGraphViewOps, prelude::*}; use raphtory_api::core::entities::{properties::meta::PropMapper, LayerId, LayerIds}; -use std::collections::{hash_map::Entry, HashMap, HashSet}; +use std::{ + collections::{hash_map::Entry, HashMap, HashSet}, + sync::Arc, +}; /// Describes a single edge layer: its name and the edge property/metadata keys /// present in it, with observed variants (property values) on small graphs @@ -13,11 +19,17 @@ use std::collections::{hash_map::Entry, HashMap, HashSet}; pub(crate) struct LayerSchema { graph: G, layer_id: LayerId, + // schema cache for the base graph, `None` for filtered/derived views + cache: Option>, } impl LayerSchema { - pub fn new(graph: G, layer_id: LayerId) -> Self { - Self { graph, layer_id } + pub fn new(graph: G, layer_id: LayerId, cache: Option>) -> Self { + Self { + graph, + layer_id, + cache, + } } } @@ -30,9 +42,14 @@ impl LayerSchema { /// Returns the list of property schemas present on edges in this layer async fn properties(&self) -> Vec { + if let Some(cache) = &self.cache { + if let Some(hit) = cache.layer().get_properties(&self.layer_id) { + return hit; + } + } let graph = self.graph.clone(); let layer_id = self.layer_id; - blocking_compute(move || { + let result = blocking_compute(move || { if too_many_edges(&graph, layer_id) { // too many edges to collect values: keys/types only, no variants return collect_layer_schema(&graph, layer_id, false); @@ -42,14 +59,25 @@ impl LayerSchema { let mapper = graph.edge_meta().temporal_prop_mapper(); collect_variants(layered.edges().into_iter().map(|e| e.properties()), mapper) }) - .await + .await; + if let Some(cache) = &self.cache { + cache + .layer() + .store_properties(self.layer_id, result.clone()); + } + result } /// Returns the list of metadata schemas present on edges in this layer async fn metadata(&self) -> Vec { + if let Some(cache) = &self.cache { + if let Some(hit) = cache.layer().get_metadata(&self.layer_id) { + return hit; + } + } let graph = self.graph.clone(); let layer_id = self.layer_id; - blocking_compute(move || { + let result = blocking_compute(move || { if too_many_edges(&graph, layer_id) { return collect_layer_schema(&graph, layer_id, true); } @@ -58,7 +86,11 @@ impl LayerSchema { let mapper = graph.edge_meta().metadata_mapper(); collect_variants(layered.edges().into_iter().map(|e| e.metadata()), mapper) }) - .await + .await; + if let Some(cache) = &self.cache { + cache.layer().store_metadata(self.layer_id, result.clone()); + } + result } } diff --git a/raphtory-graphql/src/model/schema/node_schema.rs b/raphtory-graphql/src/model/schema/node_schema.rs index 8fde5a57c1..c555d6809e 100644 --- a/raphtory-graphql/src/model/schema/node_schema.rs +++ b/raphtory-graphql/src/model/schema/node_schema.rs @@ -1,7 +1,6 @@ use crate::model::schema::{ - cache::{NodeSchemaKey, SchemaCache}, - property_schema::PropertySchema, - DEFAULT_NODE_TYPE, MAX_DETAILED_SCHEMA_ENTITIES, + cache::SchemaCache, property_schema::PropertySchema, DEFAULT_NODE_TYPE, + MAX_DETAILED_SCHEMA_ENTITIES, }; use dynamic_graphql::{ResolvedObject, ResolvedObjectFields}; use raphtory::{ @@ -26,7 +25,7 @@ use std::sync::Arc; #[derive(Clone, ResolvedObject)] pub(crate) struct NodeSchema { /// node type id; also the schema-cache lookup key - key: NodeSchemaKey, + type_id: usize, graph: DynamicGraph, // schema cache for the base graph, `None` for filtered/derived views cache: Option>, @@ -35,7 +34,7 @@ pub(crate) struct NodeSchema { impl NodeSchema { pub fn new(node_type: usize, graph: DynamicGraph, cache: Option>) -> Self { Self { - key: NodeSchemaKey::new(node_type), + type_id: node_type, graph, cache, } @@ -55,16 +54,14 @@ impl NodeSchema { /// string-valued properties) the set of distinct values. async fn properties(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.node().get_properties(&self.key) { + if let Some(hit) = cache.node().get_properties(self.type_id) { return hit; } } // not found in cache, so calculate it and cache it let result = self.properties_inner(); if let Some(cache) = &self.cache { - cache - .node() - .store_properties(self.key.clone(), result.clone()); + cache.node().store_properties(self.type_id, result.clone()); } result } @@ -73,16 +70,14 @@ impl NodeSchema { /// covering metadata fields rather than temporal properties. async fn metadata(&self) -> Vec { if let Some(cache) = &self.cache { - if let Some(hit) = cache.node().get_metadata(&self.key) { + if let Some(hit) = cache.node().get_metadata(self.type_id) { return hit; } } // not found in cache, so calculate it and cache it let result = self.metadata_inner(); if let Some(cache) = &self.cache { - cache - .node() - .store_metadata(self.key.clone(), result.clone()); + cache.node().store_metadata(self.type_id, result.clone()); } result } @@ -92,7 +87,7 @@ impl NodeSchema { fn type_name_inner(&self) -> String { self.graph .node_meta() - .get_node_type_name_by_id(self.key.type_id) + .get_node_type_name_by_id(self.type_id) .map(|type_name| type_name.to_string()) .unwrap_or_else(|| DEFAULT_NODE_TYPE.to_string()) } @@ -121,7 +116,7 @@ impl NodeSchema { .filter_map(|(key, dtype)| { let mut node_types_filter = vec![false; self.graph.node_meta().node_type_meta().num_all_fields()]; - node_types_filter[self.key.type_id] = true; + node_types_filter[self.type_id] = true; let filter = TypeId.mask(node_types_filter.into()); let unique_values: ahash::HashSet<_> = NodeFilteredGraph::new(self.graph.clone(), filter) @@ -171,7 +166,7 @@ impl NodeSchema { .filter_map(|(key, dtype)| { let mut node_types_filter = vec![false; self.graph.node_meta().node_type_meta().num_all_fields()]; - node_types_filter[self.key.type_id] = true; + node_types_filter[self.type_id] = true; let filter = TypeId.mask(node_types_filter.into()); let unique_values: ahash::HashSet<_> = NodeFilteredGraph::new(self.graph.clone(), filter) From ee63177cd5b9a55183cf2cff0d2c231af1239051 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Thu, 2 Jul 2026 11:04:38 -0400 Subject: [PATCH 22/27] regenerate schema.graphql --- raphtory-graphql/schema.graphql | 37 +++++++-------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/raphtory-graphql/schema.graphql b/raphtory-graphql/schema.graphql index dfaf911edc..e2e85eb78d 100644 --- a/raphtory-graphql/schema.graphql +++ b/raphtory-graphql/schema.graphql @@ -720,30 +720,6 @@ input EdgeLayersExpr { expr: EdgeFilter! } -""" -Describes edges between a specific pair of node types — the property and -metadata keys seen on such edges, along with their observed value types. -One `EdgeSchema` per `(srcType, dstType)` pair per layer. -""" -type EdgeSchema { - """ - Returns the type of source for these edges - """ - srcType: String! - """ - Returns the type of destination for these edges - """ - dstType: String! - """ - Returns the list of property schemas for edges connecting these types of nodes - """ - properties: [PropertySchema!]! - """ - Returns the list of metadata schemas for edges connecting these types of nodes - """ - metadata: [PropertySchema!]! -} - input EdgeSortBy { """ Reverse order @@ -2547,8 +2523,8 @@ type Intervals { } """ -Describes a single edge layer — its name and the per `(srcType, dstType)` -edge schemas observed within it. +Describes a single edge layer: its name and the edge property/metadata keys +present in it, with observed variants (property values) on small graphs """ type LayerSchema { """ @@ -2556,9 +2532,13 @@ type LayerSchema { """ name: String! """ - Returns the list of edge schemas for this edge layer + Returns the list of property schemas present on edges in this layer """ - edges: [EdgeSchema!]! + properties: [PropertySchema!]! + """ + Returns the list of metadata schemas present on edges in this layer + """ + metadata: [PropertySchema!]! } """ @@ -5644,4 +5624,3 @@ schema { query: QueryRoot mutation: MutRoot } - From ad946b4064e76884cb16e61fef9f920af7dd62ba Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Mon, 6 Jul 2026 04:02:18 -0400 Subject: [PATCH 23/27] Cleanup --- db4-storage/src/pages/edge_page/writer.rs | 9 ++++----- db4-storage/src/pages/node_page/writer.rs | 8 ++++---- db4-storage/src/properties/mod.rs | 2 +- db4-storage/src/segments/mod.rs | 1 + raphtory-api/src/core/entities/properties/meta.rs | 4 ++-- raphtory-graphql/src/model/graph/graph.rs | 4 ++-- raphtory-graphql/src/model/schema/graph_schema.rs | 4 ++-- raphtory-graphql/src/model/schema/layer_schema.rs | 9 ++------- raphtory-storage/src/graph/edges/edge_storage_ops.rs | 4 ++-- raphtory-storage/src/graph/edges/edges.rs | 4 ++-- raphtory-storage/src/graph/nodes/node_storage_ops.rs | 8 ++++---- 11 files changed, 26 insertions(+), 31 deletions(-) diff --git a/db4-storage/src/pages/edge_page/writer.rs b/db4-storage/src/pages/edge_page/writer.rs index 21a4e0d015..9c613d7a2f 100644 --- a/db4-storage/src/pages/edge_page/writer.rs +++ b/db4-storage/src/pages/edge_page/writer.rs @@ -57,9 +57,8 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen layer_id: LayerId, ) -> LocalPOS { self.graph_stats.update_time(t.t()); - // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // Update the per-layer property presence bitset in Meta. // `.inspect` runs once per emitted item as the iterator is consumed in `insert_edge_internal` - // without needing to pass Meta to insert_edge_internal let meta = self.writer.edge_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.temporal_prop_mapper() @@ -140,7 +139,7 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen } } - // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. + // Update the per-layer property presence bitset in Meta. // `.inspect` runs once per emitted item as the iterator is consumed in `insert_edge_internal` let meta = self.writer.edge_meta().clone(); let t_meta = meta.as_ref(); @@ -233,8 +232,8 @@ impl<'a, MP: DerefMut + std::fmt::Debug, ES: EdgeSegmen if !existing_edge { self.increment_layer_num_edges(layer_id); } - // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed (in update_const_properties) + // Update the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed in `update_const_properties` let meta = self.writer.edge_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); diff --git a/db4-storage/src/pages/node_page/writer.rs b/db4-storage/src/pages/node_page/writer.rs index 4cee0e6b30..c79a813541 100644 --- a/db4-storage/src/pages/node_page/writer.rs +++ b/db4-storage/src/pages/node_page/writer.rs @@ -155,8 +155,8 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri props: impl IntoIterator, ) { self.l_counter.update_time(t.t()); - // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed in add_props + // Update the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed in `add_props` let meta = self.mut_segment.node_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.temporal_prop_mapper() @@ -185,8 +185,8 @@ impl<'a, MP: DerefMut + 'a, NS: NodeSegmentOps> NodeWri layer_id: LayerId, props: impl IntoIterator, ) { - // Mirror each (layer, prop_id) into the per-layer property presence bitset in Meta. - // `.inspect` runs once per emitted item as the iterator is consumed in update_metadata + // Update the per-layer property presence bitset in Meta. + // `.inspect` runs once per emitted item as the iterator is consumed in `update_metadata` let meta = self.mut_segment.node_meta().clone(); let props = props.into_iter().inspect(move |(id, _)| { meta.metadata_mapper().mark_prop_in_layer(layer_id, *id); diff --git a/db4-storage/src/properties/mod.rs b/db4-storage/src/properties/mod.rs index 1adb9755a4..75ee01c654 100644 --- a/db4-storage/src/properties/mod.rs +++ b/db4-storage/src/properties/mod.rs @@ -23,7 +23,7 @@ use std::sync::Arc; pub mod props_meta_writer; -// Held by SegmentContainer, which corresponds to one layer in a Mem(Edge/Node)Segment +/// Contains the properties of a `SegmentContainer` (which corresponds to one layer in a Mem(Edge/Node)Segment) #[derive(Debug, Default)] pub struct Properties { c_properties: Vec, diff --git a/db4-storage/src/segments/mod.rs b/db4-storage/src/segments/mod.rs index 222c8b2d71..6c08d54b17 100644 --- a/db4-storage/src/segments/mod.rs +++ b/db4-storage/src/segments/mod.rs @@ -153,6 +153,7 @@ impl SparseVec { } } +/// A single layer's data in a Mem(Edge/Node)Segment. #[derive(Debug)] pub struct SegmentContainer { segment_id: usize, diff --git a/raphtory-api/src/core/entities/properties/meta.rs b/raphtory-api/src/core/entities/properties/meta.rs index 9dba56934d..c0cbecb4a0 100644 --- a/raphtory-api/src/core/entities/properties/meta.rs +++ b/raphtory-api/src/core/entities/properties/meta.rs @@ -319,7 +319,7 @@ impl PropMapper { .unwrap_or(false) } - /// Mark `prop_id` as present in `layer_id`. only takes the write lock once per (layer, prop) + /// Mark `prop_id` as present in `layer_id`. Only takes the write lock once per (layer, prop) pub fn mark_prop_in_layer(&self, layer_id: LayerId, prop_id: usize) { if self.layer_has(layer_id, prop_id) { return; @@ -483,7 +483,7 @@ pub struct WriteLockedPropMapper<'a> { /// Estimated size in bytes of a single row of properties maintained by this mapper. row_size: &'a AtomicUsize, - /// Per-layer property presence bitset + /// Per-layer property presence bitset. layer_presence: RwLockWriteGuard<'a, Vec>>, } diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index 0957ee8fa4..59b02c2694 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -62,8 +62,8 @@ use std::{ pub(crate) struct GqlGraph { path: UnlockedGraphFolder, graph: DynamicGraph, - // Shared schema cache, set only for the unfiltered native base graph. - // Any derived view (window/layer/filter/...) drops it so it recomputes. + /// Shared schema cache, set only for the unfiltered native base graph. + /// Any derived view (window/layer/filter/...) drops it so it recomputes. schema_cache: Option>, } diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index 83d6e6fe72..ea95c4fb9e 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -24,8 +24,8 @@ impl GraphSchema { .collect(); let layers = graph - .layer_ids() - .iter(graph.num_layers()) + .layer_ids() // FIXME: Do we want to use unique_layers() here instead? + .iter(graph.num_layers() + 1) // skip the internal `_static_graph` layer .filter(|&layer_id| layer_id != STATIC_GRAPH_LAYER_ID) .map(|layer_id| LayerSchema::new(graph.clone(), layer_id, cache.clone())) diff --git a/raphtory-graphql/src/model/schema/layer_schema.rs b/raphtory-graphql/src/model/schema/layer_schema.rs index 8696beb3dc..ea2475871b 100644 --- a/raphtory-graphql/src/model/schema/layer_schema.rs +++ b/raphtory-graphql/src/model/schema/layer_schema.rs @@ -50,7 +50,7 @@ impl LayerSchema { let graph = self.graph.clone(); let layer_id = self.layer_id; let result = blocking_compute(move || { - if too_many_edges(&graph, layer_id) { + if graph.unfiltered_num_edges(&LayerIds::One(layer_id)) > MAX_DETAILED_SCHEMA_ENTITIES { // too many edges to collect values: keys/types only, no variants return collect_layer_schema(&graph, layer_id, false); } @@ -78,7 +78,7 @@ impl LayerSchema { let graph = self.graph.clone(); let layer_id = self.layer_id; let result = blocking_compute(move || { - if too_many_edges(&graph, layer_id) { + if graph.unfiltered_num_edges(&LayerIds::One(layer_id)) > MAX_DETAILED_SCHEMA_ENTITIES { return collect_layer_schema(&graph, layer_id, true); } let layer = graph.get_layer_name(layer_id); @@ -94,11 +94,6 @@ impl LayerSchema { } } -/// True if the layer holds more edges than we're willing to scan for values. -fn too_many_edges(graph: &G, layer_id: LayerId) -> bool { - graph.unfiltered_num_edges(&LayerIds::One(layer_id)) > MAX_DETAILED_SCHEMA_ENTITIES -} - /// Get edge property/metadata keys and types using bitset without collecting values. /// Redacted properties are handled by the GraphView (in edge_layer_has_*). fn collect_layer_schema( diff --git a/raphtory-storage/src/graph/edges/edge_storage_ops.rs b/raphtory-storage/src/graph/edges/edge_storage_ops.rs index dc67e01964..2ecd2559c6 100644 --- a/raphtory-storage/src/graph/edges/edge_storage_ops.rs +++ b/raphtory-storage/src/graph/edges/edge_storage_ops.rs @@ -238,7 +238,7 @@ impl<'a> EdgeStorageOps<'a> for storage::EdgeEntryRef<'a> { EdgeRefOps::c_prop(self, layer_id, prop_id) } - /// Layer-skip override: drop layers where we know the property isn't present + // Layer-skip override: drop layers where we know the property isn't present fn temporal_prop_iter( self, layer_ids: &'a LayerIds, @@ -250,7 +250,7 @@ impl<'a> EdgeStorageOps<'a> for storage::EdgeEntryRef<'a> { .map(move |id| (id, EdgeStorageOps::temporal_prop_layer(self, id, prop_id))) } - /// Layer-skip override: same shape as `temporal_prop_iter` but against the metadata mapper + // Layer-skip override: same shape as `temporal_prop_iter` but against the metadata mapper fn metadata_iter( self, layer_ids: &'a LayerIds, diff --git a/raphtory-storage/src/graph/edges/edges.rs b/raphtory-storage/src/graph/edges/edges.rs index 9689b8c14d..a1f0004da7 100644 --- a/raphtory-storage/src/graph/edges/edges.rs +++ b/raphtory-storage/src/graph/edges/edges.rs @@ -148,7 +148,7 @@ impl<'a> EdgesStorageRef<'a> { } } - /// O(1) check via the per-layer bitset cached on `Meta.temporal_prop_mapper`. + /// O(1) check: has property `prop_id` ever been observed in `layer_id`? /// `false` is authoritative i.e. callers can skip column reads for `(layer_id, prop_id)`. pub fn layer_has_temporal_prop(&self, layer_id: LayerId, prop_id: usize) -> bool { let inner = match self { @@ -161,7 +161,7 @@ impl<'a> EdgesStorageRef<'a> { .layer_has(layer_id, prop_id) } - /// O(1) check via the per-layer bitset cached on `Meta.metadata_mapper`. + /// O(1) check: has metadata `prop_id` ever been observed in `layer_id`? /// `false` is authoritative i.e. callers can skip column reads for `(layer_id, prop_id)`. pub fn layer_has_metadata(&self, layer_id: LayerId, prop_id: usize) -> bool { let inner = match self { diff --git a/raphtory-storage/src/graph/nodes/node_storage_ops.rs b/raphtory-storage/src/graph/nodes/node_storage_ops.rs index b6ef57becc..be4183c608 100644 --- a/raphtory-storage/src/graph/nodes/node_storage_ops.rs +++ b/raphtory-storage/src/graph/nodes/node_storage_ops.rs @@ -189,8 +189,8 @@ impl<'a> NodeStorageOps<'a> for NodeEntryRef<'a> { NodeRefOps::c_prop(self, layer_id, prop_id) } - /// Layer-skip override: drop layers whose per-layer property presence bitset in - /// Meta says `prop_id` has never been written + // Layer-skip override: drop layers whose per-layer property presence bitset in + // Meta says `prop_id` has never been written fn temporal_prop_iter( self, layer_ids: &'a LayerIds, @@ -207,7 +207,7 @@ impl<'a> NodeStorageOps<'a> for NodeEntryRef<'a> { }) } - /// Layer-skip override: same shape as `temporal_prop_iter` but for the metadata mapper. + // Layer-skip override: same shape as `temporal_prop_iter` but for the metadata mapper. fn constant_prop_iter( self, layer_ids: &'a LayerIds, @@ -224,7 +224,7 @@ impl<'a> NodeStorageOps<'a> for NodeEntryRef<'a> { }) } - /// Layer-skip override: drops layers that the per-layer bitset says don't carry `prop_id`. + // Layer-skip override: drops layers that the per-layer bitset says don't carry `prop_id`. fn tprop_iter_layers( self, layer_ids: &LayerIds, From 53e2dea8273b209959adbbfe00d9727b86e9cb85 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Mon, 6 Jul 2026 04:02:51 -0400 Subject: [PATCH 24/27] Add 2 tests for the property presence bitsets filtering --- .../tests/test_layer_prop_presence.rs | 136 ++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 raphtory-tests/tests/test_layer_prop_presence.rs diff --git a/raphtory-tests/tests/test_layer_prop_presence.rs b/raphtory-tests/tests/test_layer_prop_presence.rs new file mode 100644 index 0000000000..4748ed628e --- /dev/null +++ b/raphtory-tests/tests/test_layer_prop_presence.rs @@ -0,0 +1,136 @@ +//! Round-trip tests for the per-layer property presence bitset on `Meta`. + +use raphtory::{errors::GraphError, prelude::*}; +use raphtory_api::core::entities::{properties::meta::STATIC_GRAPH_LAYER_ID, LayerId}; +use raphtory_storage::core_ops::CoreGraphOps; +use tempfile::TempDir; + +const LAYER_A: &str = "layer_a"; +const LAYER_B: &str = "layer_b"; + +/// Build a graph with properties confined to known layers: +/// - temporal edge prop `t_a` + edge metadata `m_a`: only on `LAYER_A` +/// - temporal edge prop `t_b`: only on `LAYER_B` +/// - `LAYER_B` also gets a property-less edge, so the layer exists regardless +/// - node temporal prop `np` + node metadata `nm`: nodes are unlayered, so these +/// land on the static layer +fn build_graph(dir: &TempDir) -> Result { + let g = Graph::new_at_path(dir.path())?; + + g.add_edge(0, 1, 2, [("t_a", Prop::U64(1))], Some(LAYER_A))?; + g.edge(1, 2) + .unwrap() + .add_metadata([("m_a", Prop::Bool(true))], Some(LAYER_A))?; + + g.add_edge(0, 2, 3, [("t_b", Prop::str("x"))], Some(LAYER_B))?; + g.add_edge(1, 3, 4, NO_PROPS, Some(LAYER_B))?; + + let node = g.add_node(0, 5, [("np", Prop::I64(7))], None, None)?; + node.add_metadata([("nm", Prop::str("meta"))])?; + + Ok(g) +} + +/// Assert the presence bits for the fixture built by [`build_graph`] is still consistent after reload. +fn check_presence(g: &Graph) { + let edge_meta = g.edge_meta(); + let node_meta = g.node_meta(); + + let layer_a = edge_meta.get_layer_id(LAYER_A).expect("layer_a exists"); + let layer_b = edge_meta.get_layer_id(LAYER_B).expect("layer_b exists"); + + let t_a = edge_meta + .temporal_prop_mapper() + .get_id("t_a") + .expect("t_a registered"); + let t_b = edge_meta + .temporal_prop_mapper() + .get_id("t_b") + .expect("t_b registered"); + let m_a = edge_meta + .metadata_mapper() + .get_id("m_a") + .expect("m_a registered"); + + // temporal edge props: present where written... + assert!(edge_meta.temporal_layer_has(layer_a, t_a)); + assert!(edge_meta.temporal_layer_has(layer_b, t_b)); + // ...and absent where not (guards against null-column over-marking on load) + assert!(!edge_meta.temporal_layer_has(layer_b, t_a)); + assert!(!edge_meta.temporal_layer_has(layer_a, t_b)); + + // edge metadata: only on layer_a + assert!(edge_meta.metadata_layer_has(layer_a, m_a)); + assert!(!edge_meta.metadata_layer_has(layer_b, m_a)); + + // node props/metadata land on the static layer, not the edge layers + let np = node_meta + .temporal_prop_mapper() + .get_id("np") + .expect("np registered"); + let nm = node_meta + .metadata_mapper() + .get_id("nm") + .expect("nm registered"); + assert!(node_meta.temporal_layer_has(STATIC_GRAPH_LAYER_ID, np)); + assert!(node_meta.metadata_layer_has(STATIC_GRAPH_LAYER_ID, nm)); + assert!(!node_meta.temporal_layer_has(LayerId(layer_a.0.max(layer_b.0) + 1), np)); +} + +/// The bitset must not break actual reads: values written to a layer must still +/// come back after a reload (guards against over-aggressive layer skipping). +fn check_values(g: &Graph) { + let e_a = g.edge(1, 2).unwrap(); + assert_eq!( + e_a.layers(LAYER_A).unwrap().properties().get("t_a"), + Some(Prop::U64(1)) + ); + assert_eq!( + e_a.layers(LAYER_A).unwrap().metadata().get("m_a"), + Some(Prop::Bool(true)) + ); + + let e_b = g.edge(2, 3).unwrap(); + assert_eq!( + e_b.layers(LAYER_B).unwrap().properties().get("t_b"), + Some(Prop::str("x")) + ); + // t_a lives on a different layer: the layer-skip must hide it here + assert_eq!(e_b.layers(LAYER_B).unwrap().properties().get("t_a"), None); + + let n = g.node(5).unwrap(); + assert_eq!(n.properties().get("np"), Some(Prop::I64(7))); + assert_eq!(n.metadata().get("nm"), Some(Prop::str("meta"))); +} + +/// Write -> check -> flush (mem segments persisted to arrow columns) -> drop -> load -> check. +/// The reload rebuilds `Meta` by scanning the on-disk columns so this exercises the null-gated column-scan path. +#[test] +fn layer_prop_presence_survives_flush_and_reload() { + let dir = TempDir::new().unwrap(); + + let g = build_graph(&dir).unwrap(); + check_presence(&g); + check_values(&g); + + g.core_graph().flush().unwrap(); + drop(g); + + let g = Graph::load(dir.path()).unwrap(); + check_presence(&g); + check_values(&g); +} + +/// Write -> check -> drop without an explicit flush -> load. +#[test] +fn layer_prop_presence_survives_wal_replay() { + let dir = TempDir::new().unwrap(); + + let g = build_graph(&dir).unwrap(); + check_presence(&g); + drop(g); + + let g = Graph::load(dir.path()).unwrap(); + check_presence(&g); + check_values(&g); +} From 3097a799c61ed3d9ac45b3edd2e09ddff9440947 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Tue, 7 Jul 2026 04:40:45 -0400 Subject: [PATCH 25/27] Fixed failing tests. Moved the tests for round trip persistence of the bitset to integration-tests (in pometry-storage) instead of raphtory-tests (in Raphtory). Fixed queries and responses to match new schemas. --- .../test_graphql/test_schema.py | 202 +++++++----------- .../tests/test_layer_prop_presence.rs | 136 ------------ 2 files changed, 75 insertions(+), 263 deletions(-) delete mode 100644 raphtory-tests/tests/test_layer_prop_presence.rs diff --git a/python/tests/test_base_install/test_graphql/test_schema.py b/python/tests/test_base_install/test_graphql/test_schema.py index 6c67fd58b3..7d7ba45cc5 100644 --- a/python/tests/test_base_install/test_graphql/test_schema.py +++ b/python/tests/test_base_install/test_graphql/test_schema.py @@ -21,11 +21,6 @@ def sort_lists_in_structure(key_main, obj): (sort_lists_in_structure(None, item) for item in obj), key=lambda x: x["typeName"], ) - if key_main == "edges": - return sorted( - (sort_lists_in_structure(None, item) for item in obj), - key=lambda x: (x["srcType"], x["dstType"]), - ) if key_main == "properties": return sorted( (sort_lists_in_structure(None, item) for item in obj), @@ -66,19 +61,15 @@ def test_node_edge_properties_schema(): schema { layers { name - edges { - srcType - dstType - properties { - key - propertyType - variants - } - metadata { - key - propertyType - variants - } + properties { + key + propertyType + variants + } + metadata { + key + propertyType + variants } } nodes { @@ -92,7 +83,7 @@ def test_node_edge_properties_schema(): key propertyType variants - } + } } } } @@ -105,135 +96,92 @@ def test_node_edge_properties_schema(): "layers": [ { "name": "0", - "edges": [ + "properties": [ + { + "key": "prop1", + "propertyType": "I64", + "variants": ["1"], + }, + { + "key": "prop2", + "propertyType": "F64", + "variants": ["9.8"], + }, { - "srcType": "a", - "dstType": "None", - "properties": [ - { - "key": "prop1", - "propertyType": "I64", - "variants": ["1"], - }, - { - "key": "prop3", - "propertyType": "Str", - "variants": ["test"], - }, - ], - "metadata": [ - { - "key": "static", - "propertyType": "Str", - "variants": ["test"], - }, - ], + "key": "prop3", + "propertyType": "Str", + "variants": ["test"], }, + ], + "metadata": [ { - "srcType": "None", - "dstType": "b", - "properties": [ - { - "key": "prop1", - "propertyType": "I64", - "variants": ["1"], - }, - { - "key": "prop2", - "propertyType": "F64", - "variants": ["9.8"], - }, - ], - "metadata": [], + "key": "static", + "propertyType": "Str", + "variants": ["test"], }, ], }, { "name": "1", - "edges": [ + "properties": [ { - "srcType": "b", - "dstType": "b", - "properties": [ - { - "key": "prop4", - "propertyType": "I64", - "variants": ["1"], - }, - { - "key": "prop5", - "propertyType": "F64", - "variants": ["9.8"], - }, - { - "key": "prop6", - "propertyType": "Map{ data: Str }", - "variants": ['{"data": "map"}'], - }, - ], - "metadata": [], + "key": "prop1", + "propertyType": "I64", + "variants": ["1"], }, { - "srcType": "b", - "dstType": "None", - "properties": [ - { - "key": "prop1", - "propertyType": "I64", - "variants": ["1"], - }, - { - "key": "propArray", - "propertyType": "List", - "variants": ["[1, 2, 3]"], - }, - { - "key": "prop2", - "propertyType": "F64", - "variants": ["9.8"], - }, - ], - "metadata": [], + "key": "prop2", + "propertyType": "F64", + "variants": ["9.8"], + }, + { + "key": "prop4", + "propertyType": "I64", + "variants": ["1"], + }, + { + "key": "prop5", + "propertyType": "F64", + "variants": ["9.8"], + }, + { + "key": "prop6", + "propertyType": "Map{ data: Str }", + "variants": ['{"data": "map"}'], + }, + { + "key": "propArray", + "propertyType": "List", + "variants": ["[1, 2, 3]"], }, ], + "metadata": [], }, { "name": "2", - "edges": [ + "properties": [ { - "srcType": "None", - "dstType": "None", - "properties": [ - { - "key": "prop1", - "propertyType": "I64", - "variants": ["1"], - }, - { - "key": "prop2", - "propertyType": "F64", - "variants": ["9.8"], - }, - { - "key": "prop3", - "propertyType": "Str", - "variants": ["test"], - }, - ], - "metadata": [], - } + "key": "prop1", + "propertyType": "I64", + "variants": ["1"], + }, + { + "key": "prop2", + "propertyType": "F64", + "variants": ["9.8"], + }, + { + "key": "prop3", + "propertyType": "Str", + "variants": ["test"], + }, ], + "metadata": [], }, { "name": "3", - "edges": [ - { - "srcType": "None", - "dstType": "None", - "properties": [], - "metadata": [], - } - ], + "properties": [], + "metadata": [], }, ], "nodes": [ diff --git a/raphtory-tests/tests/test_layer_prop_presence.rs b/raphtory-tests/tests/test_layer_prop_presence.rs deleted file mode 100644 index 4748ed628e..0000000000 --- a/raphtory-tests/tests/test_layer_prop_presence.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Round-trip tests for the per-layer property presence bitset on `Meta`. - -use raphtory::{errors::GraphError, prelude::*}; -use raphtory_api::core::entities::{properties::meta::STATIC_GRAPH_LAYER_ID, LayerId}; -use raphtory_storage::core_ops::CoreGraphOps; -use tempfile::TempDir; - -const LAYER_A: &str = "layer_a"; -const LAYER_B: &str = "layer_b"; - -/// Build a graph with properties confined to known layers: -/// - temporal edge prop `t_a` + edge metadata `m_a`: only on `LAYER_A` -/// - temporal edge prop `t_b`: only on `LAYER_B` -/// - `LAYER_B` also gets a property-less edge, so the layer exists regardless -/// - node temporal prop `np` + node metadata `nm`: nodes are unlayered, so these -/// land on the static layer -fn build_graph(dir: &TempDir) -> Result { - let g = Graph::new_at_path(dir.path())?; - - g.add_edge(0, 1, 2, [("t_a", Prop::U64(1))], Some(LAYER_A))?; - g.edge(1, 2) - .unwrap() - .add_metadata([("m_a", Prop::Bool(true))], Some(LAYER_A))?; - - g.add_edge(0, 2, 3, [("t_b", Prop::str("x"))], Some(LAYER_B))?; - g.add_edge(1, 3, 4, NO_PROPS, Some(LAYER_B))?; - - let node = g.add_node(0, 5, [("np", Prop::I64(7))], None, None)?; - node.add_metadata([("nm", Prop::str("meta"))])?; - - Ok(g) -} - -/// Assert the presence bits for the fixture built by [`build_graph`] is still consistent after reload. -fn check_presence(g: &Graph) { - let edge_meta = g.edge_meta(); - let node_meta = g.node_meta(); - - let layer_a = edge_meta.get_layer_id(LAYER_A).expect("layer_a exists"); - let layer_b = edge_meta.get_layer_id(LAYER_B).expect("layer_b exists"); - - let t_a = edge_meta - .temporal_prop_mapper() - .get_id("t_a") - .expect("t_a registered"); - let t_b = edge_meta - .temporal_prop_mapper() - .get_id("t_b") - .expect("t_b registered"); - let m_a = edge_meta - .metadata_mapper() - .get_id("m_a") - .expect("m_a registered"); - - // temporal edge props: present where written... - assert!(edge_meta.temporal_layer_has(layer_a, t_a)); - assert!(edge_meta.temporal_layer_has(layer_b, t_b)); - // ...and absent where not (guards against null-column over-marking on load) - assert!(!edge_meta.temporal_layer_has(layer_b, t_a)); - assert!(!edge_meta.temporal_layer_has(layer_a, t_b)); - - // edge metadata: only on layer_a - assert!(edge_meta.metadata_layer_has(layer_a, m_a)); - assert!(!edge_meta.metadata_layer_has(layer_b, m_a)); - - // node props/metadata land on the static layer, not the edge layers - let np = node_meta - .temporal_prop_mapper() - .get_id("np") - .expect("np registered"); - let nm = node_meta - .metadata_mapper() - .get_id("nm") - .expect("nm registered"); - assert!(node_meta.temporal_layer_has(STATIC_GRAPH_LAYER_ID, np)); - assert!(node_meta.metadata_layer_has(STATIC_GRAPH_LAYER_ID, nm)); - assert!(!node_meta.temporal_layer_has(LayerId(layer_a.0.max(layer_b.0) + 1), np)); -} - -/// The bitset must not break actual reads: values written to a layer must still -/// come back after a reload (guards against over-aggressive layer skipping). -fn check_values(g: &Graph) { - let e_a = g.edge(1, 2).unwrap(); - assert_eq!( - e_a.layers(LAYER_A).unwrap().properties().get("t_a"), - Some(Prop::U64(1)) - ); - assert_eq!( - e_a.layers(LAYER_A).unwrap().metadata().get("m_a"), - Some(Prop::Bool(true)) - ); - - let e_b = g.edge(2, 3).unwrap(); - assert_eq!( - e_b.layers(LAYER_B).unwrap().properties().get("t_b"), - Some(Prop::str("x")) - ); - // t_a lives on a different layer: the layer-skip must hide it here - assert_eq!(e_b.layers(LAYER_B).unwrap().properties().get("t_a"), None); - - let n = g.node(5).unwrap(); - assert_eq!(n.properties().get("np"), Some(Prop::I64(7))); - assert_eq!(n.metadata().get("nm"), Some(Prop::str("meta"))); -} - -/// Write -> check -> flush (mem segments persisted to arrow columns) -> drop -> load -> check. -/// The reload rebuilds `Meta` by scanning the on-disk columns so this exercises the null-gated column-scan path. -#[test] -fn layer_prop_presence_survives_flush_and_reload() { - let dir = TempDir::new().unwrap(); - - let g = build_graph(&dir).unwrap(); - check_presence(&g); - check_values(&g); - - g.core_graph().flush().unwrap(); - drop(g); - - let g = Graph::load(dir.path()).unwrap(); - check_presence(&g); - check_values(&g); -} - -/// Write -> check -> drop without an explicit flush -> load. -#[test] -fn layer_prop_presence_survives_wal_replay() { - let dir = TempDir::new().unwrap(); - - let g = build_graph(&dir).unwrap(); - check_presence(&g); - drop(g); - - let g = Graph::load(dir.path()).unwrap(); - check_presence(&g); - check_values(&g); -} From a7db749652af121c5d30c15dee32761e9fb10110 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Tue, 7 Jul 2026 12:11:57 +0000 Subject: [PATCH 26/27] chore: apply tidy-public auto-fixes --- docs/reference/graphql/graphql_API.md | 74 +++++---------------------- raphtory-graphql/schema.graphql | 1 + 2 files changed, 15 insertions(+), 60 deletions(-) diff --git a/docs/reference/graphql/graphql_API.md b/docs/reference/graphql/graphql_API.md index 3f85bd9b5c..1a6b438895 100644 --- a/docs/reference/graphql/graphql_API.md +++ b/docs/reference/graphql/graphql_API.md @@ -1505,61 +1505,6 @@ Composite edge filter (by property, layer, src/dst, etc.). -### EdgeSchema - -Describes edges between a specific pair of node types — the property and -metadata keys seen on such edges, along with their observed value types. -One `EdgeSchema` per `(srcType, dstType)` pair per layer. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FieldArgumentTypeDescription
srcTypeString! - -Returns the type of source for these edges - -
dstTypeString! - -Returns the type of destination for these edges - -
properties[PropertySchema!]! - -Returns the list of property schemas for edges connecting these types of nodes - -
metadata[PropertySchema!]! - -Returns the list of metadata schemas for edges connecting these types of nodes - -
- ### EdgeWindowSet A lazy sequence of per-window views of a single edge, produced by @@ -4297,8 +4242,8 @@ Compute the minimum interval between consecutive timestamps. Returns None if few ### LayerSchema -Describes a single edge layer — its name and the per `(srcType, dstType)` -edge schemas observed within it. +Describes a single edge layer: its name and the edge property/metadata keys +present in it, with observed variants (property values) on small graphs @@ -4320,11 +4265,20 @@ Returns the name of the layer with this schema - - + + + + + + + diff --git a/raphtory-graphql/schema.graphql b/raphtory-graphql/schema.graphql index e2e85eb78d..7f63354f72 100644 --- a/raphtory-graphql/schema.graphql +++ b/raphtory-graphql/schema.graphql @@ -5624,3 +5624,4 @@ schema { query: QueryRoot mutation: MutRoot } + From 3df6924691e8da61025883651e80c1393c1be951 Mon Sep 17 00:00:00 2001 From: arienandalibi Date: Wed, 8 Jul 2026 03:12:34 -0400 Subject: [PATCH 27/27] Change logic to get layer ids from the graph. We use unique_layers now. --- raphtory-graphql/src/graph.rs | 6 +++--- raphtory-graphql/src/model/schema/graph_schema.rs | 13 ++++++------- .../view/internal/time_semantics/filtered_edge.rs | 5 +---- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/raphtory-graphql/src/graph.rs b/raphtory-graphql/src/graph.rs index d26b083fc6..816e60a154 100644 --- a/raphtory-graphql/src/graph.rs +++ b/raphtory-graphql/src/graph.rs @@ -48,8 +48,8 @@ pub struct GraphWithVectorsInner { pub is_dirty: AtomicBool, pub is_flushing: AtomicBool, /// Cache of computed edge schemas for the unfiltered base view of this graph. - /// Cleared on every mutation - pub schema_cache: Arc, + /// Cleared on every mutation. + pub(crate) schema_cache: Arc, } impl GraphWithVectors { @@ -103,7 +103,7 @@ impl GraphWithVectors { } /// Handle to this graph's schema cache. - pub fn schema_cache(&self) -> Arc { + pub(crate) fn schema_cache(&self) -> Arc { self.inner.schema_cache.clone() } diff --git a/raphtory-graphql/src/model/schema/graph_schema.rs b/raphtory-graphql/src/model/schema/graph_schema.rs index ea95c4fb9e..7b231268e3 100644 --- a/raphtory-graphql/src/model/schema/graph_schema.rs +++ b/raphtory-graphql/src/model/schema/graph_schema.rs @@ -4,8 +4,7 @@ use crate::model::schema::{ use dynamic_graphql::SimpleObject; use itertools::Itertools; use raphtory::{db::api::view::DynamicGraph, prelude::*}; -use raphtory_api::core::entities::properties::meta::STATIC_GRAPH_LAYER_ID; -use raphtory_storage::{core_ops::CoreGraphOps, layer_ops::InternalLayerOps}; +use raphtory_storage::core_ops::CoreGraphOps; use std::sync::Arc; #[derive(SimpleObject)] @@ -24,11 +23,11 @@ impl GraphSchema { .collect(); let layers = graph - .layer_ids() // FIXME: Do we want to use unique_layers() here instead? - .iter(graph.num_layers() + 1) - // skip the internal `_static_graph` layer - .filter(|&layer_id| layer_id != STATIC_GRAPH_LAYER_ID) - .map(|layer_id| LayerSchema::new(graph.clone(), layer_id, cache.clone())) + .unique_layers() + .filter_map(|layer| { + let layer_id = graph.get_layer_id(layer.as_ref())?; + Some(LayerSchema::new(graph.clone(), layer_id, cache.clone())) + }) .collect_vec(); GraphSchema { nodes, layers } diff --git a/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs b/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs index 6dc8f98dac..3ad8bd6704 100644 --- a/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs +++ b/raphtory/src/db/api/view/internal/time_semantics/filtered_edge.rs @@ -1,8 +1,5 @@ use crate::{ - db::api::{ - properties::internal::EdgePropertySchemaOps, - view::internal::{FilterOps, FilterState, FilterVariants, GraphView}, - }, + db::api::view::internal::{FilterOps, FilterState, FilterVariants, GraphView}, prelude::{GraphViewOps, LayerOps}, }; use either::Either;
edges[EdgeSchema!]!properties[PropertySchema!]! + +Returns the list of property schemas present on edges in this layer + +
metadata[PropertySchema!]! -Returns the list of edge schemas for this edge layer +Returns the list of metadata schemas present on edges in this layer