From 79bc0911d75dd5dae31658b717c0fa26a0a30546 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Fri, 2 Dec 2022 12:22:24 +0800 Subject: [PATCH 1/8] [GIE/Runtime] Support `Entry` as Trait, and Impl `Entry` for some common used type. Fix CI Tests. --- .../executor/common/dyn_type/src/object.rs | 4 + .../src/adapters/exp_store/read_graph.rs | 2 - .../src/adapters/gs_store/read_graph.rs | 2 - .../src/apis/graph/element/edge.rs | 5 + .../graph_proxy/src/apis/graph/element/mod.rs | 2 +- .../src/apis/graph/element/path.rs | 174 +---- .../src/apis/graph/element/vertex.rs | 7 +- .../executor/ir/graph_proxy/src/apis/mod.rs | 1 - .../executor/ir/graph_proxy/src/lib.rs | 2 + .../ir/graph_proxy/src/utils/expr/eval.rs | 76 ++- .../graph_proxy/src/utils/expr/eval_pred.rs | 36 +- .../ir/integrated/tests/apply_test.rs | 27 +- .../ir/integrated/tests/auxilia_test.rs | 1 + .../ir/integrated/tests/common/mod.rs | 9 +- .../ir/integrated/tests/expand_test.rs | 54 +- .../ir/integrated/tests/graph_query_test.rs | 12 +- .../ir/integrated/tests/match_test.rs | 1 + .../ir/integrated/tests/pathxd_test.rs | 1 + .../executor/ir/integrated/tests/scan_test.rs | 1 + .../executor/ir/runtime/src/error.rs | 3 + .../executor/ir/runtime/src/process/entry.rs | 636 ++++++++++++++++++ .../executor/ir/runtime/src/process/mod.rs | 1 + .../src/process/operator/accum/accum.rs | 157 ++--- .../src/process/operator/filter/select.rs | 1 + .../process/operator/flatmap/edge_expand.rs | 94 +-- .../src/process/operator/flatmap/get_v.rs | 1 + .../src/process/operator/flatmap/unfold.rs | 40 +- .../src/process/operator/group/fold.rs | 21 +- .../src/process/operator/group/group.rs | 100 +-- .../runtime/src/process/operator/join/join.rs | 4 +- .../src/process/operator/keyed/keyed.rs | 4 +- .../src/process/operator/map/auxilia.rs | 101 ++- .../process/operator/map/expand_intersect.rs | 129 +++- .../runtime/src/process/operator/map/get_v.rs | 1 + .../src/process/operator/map/path_start.rs | 1 + .../src/process/operator/map/project.rs | 234 ++++--- .../ir/runtime/src/process/operator/mod.rs | 154 ++--- .../runtime/src/process/operator/shuffle.rs | 50 +- .../runtime/src/process/operator/sink/sink.rs | 73 +- .../process/operator/sink/sink_vineyard.rs | 1 + .../runtime/src/process/operator/sort/sort.rs | 1 + .../src/process/operator/subtask/apply.rs | 6 +- .../executor/ir/runtime/src/process/record.rs | 355 ++-------- 43 files changed, 1525 insertions(+), 1060 deletions(-) create mode 100644 interactive_engine/executor/ir/runtime/src/process/entry.rs diff --git a/interactive_engine/executor/common/dyn_type/src/object.rs b/interactive_engine/executor/common/dyn_type/src/object.rs index 001c5b3af92d..bdff335aeabb 100644 --- a/interactive_engine/executor/common/dyn_type/src/object.rs +++ b/interactive_engine/executor/common/dyn_type/src/object.rs @@ -14,6 +14,8 @@ //! limitations under the License. use core::any::TypeId; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; use std::any::Any; use std::borrow::Cow; use std::cmp::Ordering; @@ -418,6 +420,8 @@ pub enum Object { None, } +impl_as_any!(Object); + impl ToString for Object { fn to_string(&self) -> String { match self { diff --git a/interactive_engine/executor/ir/graph_proxy/src/adapters/exp_store/read_graph.rs b/interactive_engine/executor/ir/graph_proxy/src/adapters/exp_store/read_graph.rs index a10d5eb088a2..24fdb11f3edc 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/adapters/exp_store/read_graph.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/adapters/exp_store/read_graph.rs @@ -31,8 +31,6 @@ use ir_common::{KeyId, LabelId, NameOrId}; use pegasus::configure_with_default; use pegasus_common::downcast::*; use pegasus_common::impl_as_any; -use rand::prelude::StdRng; -use rand::{Rng, SeedableRng}; use crate::apis::graph::PKV; use crate::apis::{ diff --git a/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs b/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs index e3fb00d77665..e9612a54dcd1 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/adapters/gs_store/read_graph.rs @@ -28,8 +28,6 @@ use global_query::{ }; use graph_store::utils::IterList; use ir_common::{KeyId, LabelId, NameOrId, OneOrMany}; -use rand::prelude::StdRng; -use rand::{Rng, SeedableRng}; use super::details::{HybridEdgeDetails, HybridVertexDetails, LazyEdgeDetails, LazyVertexDetails}; use crate::apis::graph::PKV; diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/edge.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/edge.rs index 0b96081c79db..789182770431 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/edge.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/edge.rs @@ -13,6 +13,7 @@ //! See the License for the specific language governing permissions and //! limitations under the License. +use std::any::Any; use std::cmp::Ordering; use std::convert::{TryFrom, TryInto}; use std::hash::{Hash, Hasher}; @@ -23,6 +24,8 @@ use ir_common::error::ParsePbError; use ir_common::generated::results as result_pb; use ir_common::{LabelId, NameOrId}; use pegasus_common::codec::{Decode, Encode, ReadExt, WriteExt}; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; use crate::apis::{read_id, write_id, DynDetails, Element, GraphElement, ID}; use crate::utils::expr::eval::Context; @@ -40,6 +43,8 @@ pub struct Edge { details: DynDetails, } +impl_as_any!(Edge); + impl Element for Edge { fn as_graph_element(&self) -> Option<&dyn GraphElement> { Some(self) diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/mod.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/mod.rs index 39a3347f6205..fd4232167f03 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/mod.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/mod.rs @@ -17,7 +17,6 @@ use dyn_type::{BorrowObject, Object}; pub use edge::Edge; use ir_common::LabelId; pub use path::GraphPath; -pub use path::VertexOrEdge; pub use property::{Details, DynDetails, PropKey, PropertyValue}; pub use vertex::Vertex; @@ -42,6 +41,7 @@ pub trait Element { } /// `GraphElement` is a special `Element` with extra properties of `id` and `label`. +/// In common case, `GraphElement` refers to `Vertex` and `Edge` pub trait GraphElement: Element { fn id(&self) -> ID; fn label(&self) -> Option; diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs index 51b3169e94ba..06376314644c 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs @@ -13,8 +13,8 @@ //! See the License for the specific language governing permissions and //! limitations under the License. +use std::any::Any; use std::cmp::Ordering; -use std::collections::hash_map::DefaultHasher; use std::convert::{TryFrom, TryInto}; use std::hash::{Hash, Hasher}; @@ -23,64 +23,47 @@ use ir_common::error::ParsePbError; use ir_common::generated::algebra::path_expand::PathOpt; use ir_common::generated::algebra::path_expand::ResultOpt; use ir_common::generated::results as result_pb; -use ir_common::LabelId; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; -use crate::apis::{DynDetails, Edge, Element, GraphElement, Vertex, ID}; - -#[derive(Clone, Debug, Hash, PartialEq, PartialOrd)] -pub enum VertexOrEdge { - V(Vertex), - E(Edge), -} - -impl From for VertexOrEdge { - fn from(v: Vertex) -> Self { - Self::V(v) - } -} - -impl From for VertexOrEdge { - fn from(e: Edge) -> Self { - Self::E(e) - } -} +use crate::apis::{Element, GraphElement, Vertex, ID}; #[derive(Clone, Debug)] pub enum GraphPath { - AllV(Vec), - SimpleAllV(Vec), - EndV((VertexOrEdge, usize)), - SimpleEndV((VertexOrEdge, Vec)), + AllV(Vec), + SimpleAllV(Vec), + EndV((Vertex, usize)), + SimpleEndV((Vertex, Vec)), } +impl_as_any!(GraphPath); + impl GraphPath { - pub fn new>(entry: E, path_opt: PathOpt, result_opt: ResultOpt) -> Self { + pub fn new(entry: Vertex, path_opt: PathOpt, result_opt: ResultOpt) -> Self { match result_opt { ResultOpt::EndV => match path_opt { - PathOpt::Arbitrary => GraphPath::EndV((entry.into(), 1)), + PathOpt::Arbitrary => GraphPath::EndV((entry, 1)), PathOpt::Simple => { - let entry = entry.into(); let id = entry.id(); GraphPath::SimpleEndV((entry, vec![id])) } }, ResultOpt::AllV => match path_opt { - PathOpt::Arbitrary => GraphPath::AllV(vec![entry.into()]), - PathOpt::Simple => GraphPath::SimpleAllV(vec![entry.into()]), + PathOpt::Arbitrary => GraphPath::AllV(vec![entry]), + PathOpt::Simple => GraphPath::SimpleAllV(vec![entry]), }, } } // append an entry and return the flag of whether the entry has been appended or not. - pub fn append>(&mut self, entry: E) -> bool { + pub fn append(&mut self, entry: Vertex) -> bool { match self { GraphPath::AllV(ref mut path) => { - path.push(entry.into()); + path.push(entry); true } GraphPath::SimpleAllV(ref mut path) => { - let entry = entry.into(); if path.contains(&entry) { false } else { @@ -89,31 +72,30 @@ impl GraphPath { } } GraphPath::EndV((ref mut e, ref mut weight)) => { - *e = entry.into(); + *e = entry; *weight += 1; true } GraphPath::SimpleEndV((ref mut e, ref mut path)) => { - let entry = entry.into(); if path.contains(&entry.id()) { false } else { path.push(entry.id()); - *e = entry.into(); + *e = entry; true } } } } - pub fn get_path_end(&self) -> Option<&VertexOrEdge> { + pub fn get_path_end(&self) -> Option<&Vertex> { match self { GraphPath::AllV(ref p) | GraphPath::SimpleAllV(ref p) => p.last(), GraphPath::EndV((ref e, _)) | GraphPath::SimpleEndV((ref e, _)) => Some(e), } } - pub fn take_path(self) -> Option> { + pub fn take_path(self) -> Option> { match self { GraphPath::AllV(p) | GraphPath::SimpleAllV(p) => Some(p), GraphPath::EndV(_) | GraphPath::SimpleEndV(_) => None, @@ -121,54 +103,9 @@ impl GraphPath { } } -impl Element for VertexOrEdge { - fn as_graph_element(&self) -> Option<&dyn GraphElement> { - match self { - VertexOrEdge::V(v) => v.as_graph_element(), - VertexOrEdge::E(e) => e.as_graph_element(), - } - } - - fn len(&self) -> usize { - match self { - VertexOrEdge::V(v) => v.len(), - VertexOrEdge::E(e) => e.len(), - } - } - - fn as_borrow_object(&self) -> BorrowObject { - match self { - VertexOrEdge::V(v) => v.as_borrow_object(), - VertexOrEdge::E(e) => e.as_borrow_object(), - } - } -} - -impl GraphElement for VertexOrEdge { - fn id(&self) -> ID { - match self { - VertexOrEdge::V(v) => v.id(), - VertexOrEdge::E(e) => e.id(), - } - } - - fn label(&self) -> Option { - match self { - VertexOrEdge::V(v) => v.label(), - VertexOrEdge::E(e) => e.label(), - } - } - fn details(&self) -> Option<&DynDetails> { - match self { - VertexOrEdge::V(v) => v.details(), - VertexOrEdge::E(e) => e.details(), - } - } -} - impl Element for GraphPath { fn as_graph_element(&self) -> Option<&dyn GraphElement> { - Some(self) + None } // the path len is the number of edges in the path; @@ -185,24 +122,6 @@ impl Element for GraphPath { } } -impl GraphElement for GraphPath { - fn id(&self) -> ID { - match self { - GraphPath::AllV(path) | GraphPath::SimpleAllV(path) => { - let ids: Vec = path.iter().map(|v| v.id()).collect(); - let mut hasher = DefaultHasher::new(); - ids.hash(&mut hasher); - hasher.finish() as ID - } - GraphPath::EndV((path_end, _)) | GraphPath::SimpleEndV((path_end, _)) => path_end.id(), - } - } - - fn label(&self) -> Option { - None - } -} - impl PartialEq for GraphPath { fn eq(&self, other: &Self) -> bool { // We define eq by structure, ignoring path weight @@ -219,6 +138,7 @@ impl PartialEq for GraphPath { } } } + impl PartialOrd for GraphPath { // We define partial_cmp by structure, ignoring path weight fn partial_cmp(&self, other: &Self) -> Option { @@ -236,39 +156,6 @@ impl PartialOrd for GraphPath { } } -impl Encode for VertexOrEdge { - fn write_to(&self, writer: &mut W) -> std::io::Result<()> { - match self { - VertexOrEdge::V(v) => { - writer.write_u8(0)?; - v.write_to(writer)?; - } - VertexOrEdge::E(e) => { - writer.write_u8(1)?; - e.write_to(writer)?; - } - } - Ok(()) - } -} - -impl Decode for VertexOrEdge { - fn read_from(reader: &mut R) -> std::io::Result { - let e = reader.read_u8()?; - match e { - 0 => { - let v = ::read_from(reader)?; - Ok(VertexOrEdge::V(v)) - } - 1 => { - let e = ::read_from(reader)?; - Ok(VertexOrEdge::E(e)) - } - _ => Err(std::io::Error::new(std::io::ErrorKind::Other, "unreachable")), - } - } -} - impl Encode for GraphPath { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { match self { @@ -300,20 +187,20 @@ impl Decode for GraphPath { let opt = reader.read_u8()?; match opt { 0 => { - let path = >::read_from(reader)?; + let path = >::read_from(reader)?; Ok(GraphPath::AllV(path)) } 1 => { - let vertex_or_edge = ::read_from(reader)?; + let vertex_or_edge = ::read_from(reader)?; let weight = ::read_from(reader)? as usize; Ok(GraphPath::EndV((vertex_or_edge, weight))) } 2 => { - let path = >::read_from(reader)?; + let path = >::read_from(reader)?; Ok(GraphPath::SimpleAllV(path)) } 3 => { - let vertex_or_edge = ::read_from(reader)?; + let vertex_or_edge = ::read_from(reader)?; let path = >::read_from(reader)?; Ok(GraphPath::SimpleEndV((vertex_or_edge, path))) } @@ -322,7 +209,7 @@ impl Decode for GraphPath { } } -impl TryFrom for VertexOrEdge { +impl TryFrom for Vertex { type Error = ParsePbError; fn try_from(e: result_pb::graph_path::VertexOrEdge) -> Result { let vertex_or_edge = e @@ -331,11 +218,10 @@ impl TryFrom for VertexOrEdge { match vertex_or_edge { result_pb::graph_path::vertex_or_edge::Inner::Vertex(v) => { let vertex = v.try_into()?; - Ok(VertexOrEdge::V(vertex)) + Ok(vertex) } - result_pb::graph_path::vertex_or_edge::Inner::Edge(e) => { - let edge = e.try_into()?; - Ok(VertexOrEdge::E(edge)) + result_pb::graph_path::vertex_or_edge::Inner::Edge(_) => { + Err(ParsePbError::Unsupported("Path with edges".to_string())) } } } diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/vertex.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/vertex.rs index 9782a23c5bd0..9febdac15886 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/vertex.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/vertex.rs @@ -13,6 +13,7 @@ //! See the License for the specific language governing permissions and //! limitations under the License. +use std::any::Any; use std::cmp::Ordering; use std::convert::{TryFrom, TryInto}; use std::hash::{Hash, Hasher}; @@ -23,11 +24,13 @@ use ir_common::error::ParsePbError; use ir_common::generated::results as result_pb; use ir_common::{LabelId, NameOrId}; use pegasus_common::codec::{Decode, Encode, ReadExt, WriteExt}; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; use crate::apis::{read_id, write_id, DynDetails, Element, GraphElement, ID}; use crate::utils::expr::eval::Context; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Default)] pub struct Vertex { id: ID, label: Option, @@ -68,6 +71,8 @@ impl GraphElement for Vertex { } } +impl_as_any!(Vertex); + impl Encode for Vertex { fn write_to(&self, writer: &mut W) -> io::Result<()> { write_id(writer, self.id)?; diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/mod.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/mod.rs index cd54da415442..dfc1763ba166 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/mod.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/mod.rs @@ -20,7 +20,6 @@ pub mod write_graph; pub use graph::element::{ Details, DynDetails, Edge, Element, GraphElement, GraphPath, PropKey, PropertyValue, Vertex, - VertexOrEdge, }; pub use graph::{read_id, write_id, Direction, QueryParams, ID}; pub use partitioner::Partitioner; diff --git a/interactive_engine/executor/ir/graph_proxy/src/lib.rs b/interactive_engine/executor/ir/graph_proxy/src/lib.rs index 3a1cb1b1d930..2dbfcbd7fb41 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/lib.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/lib.rs @@ -59,6 +59,8 @@ macro_rules! filter_limit { macro_rules! sample_limit { ($iter: expr, $s: expr, $n: expr) => { if let Some(ratio) = $s { + use rand::prelude::StdRng; + use rand::{Rng, SeedableRng}; let mut rng: StdRng = SeedableRng::from_entropy(); let r = $iter.filter(move |_| rng.gen_bool(ratio)); limit_n!(r, $n) diff --git a/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval.rs b/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval.rs index 6b656ff03be3..24729ffb832d 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval.rs @@ -462,42 +462,46 @@ impl Evaluate for Operand { if let Some(ctxt) = context { if let Some(element) = ctxt.get(tag.as_ref()) { let result = if let Some(property) = prop_key { - let graph_element = element - .as_graph_element() - .ok_or(ExprEvalError::UnexpectedDataType(self.into()))?; - match property { - PropKey::Id => graph_element.id().into(), - PropKey::Label => graph_element - .label() - .map(|label| label.into()) - .ok_or(ExprEvalError::GetNoneFromContext)?, - PropKey::Len => graph_element.len().into(), - PropKey::All => graph_element - .details() - .ok_or(ExprEvalError::UnexpectedDataType(self.into()))? - .get_all_properties() - .map(|obj| { - obj.into_iter() - .map(|(key, value)| { - let obj_key: Object = match key { - NameOrId::Str(str) => str.into(), - NameOrId::Id(id) => id.into(), - }; - (obj_key, value) - }) - .collect::>() - .into() - }) - .ok_or(ExprEvalError::GetNoneFromContext)?, - PropKey::Key(key) => graph_element - .details() - .ok_or(ExprEvalError::UnexpectedDataType(self.into()))? - .get_property(key) - .ok_or(ExprEvalError::GetNoneFromContext)? - .try_to_owned() - .ok_or(ExprEvalError::OtherErr( - "cannot get `Object` from `BorrowObject`".to_string(), - ))?, + if let PropKey::Len = property { + element.len().into() + } else { + let graph_element = element + .as_graph_element() + .ok_or(ExprEvalError::UnexpectedDataType(self.into()))?; + match property { + PropKey::Id => graph_element.id().into(), + PropKey::Label => graph_element + .label() + .map(|label| label.into()) + .ok_or(ExprEvalError::GetNoneFromContext)?, + PropKey::Len => unreachable!(), + PropKey::All => graph_element + .details() + .ok_or(ExprEvalError::UnexpectedDataType(self.into()))? + .get_all_properties() + .map(|obj| { + obj.into_iter() + .map(|(key, value)| { + let obj_key: Object = match key { + NameOrId::Str(str) => str.into(), + NameOrId::Id(id) => id.into(), + }; + (obj_key, value) + }) + .collect::>() + .into() + }) + .ok_or(ExprEvalError::GetNoneFromContext)?, + PropKey::Key(key) => graph_element + .details() + .ok_or(ExprEvalError::UnexpectedDataType(self.into()))? + .get_property(key) + .ok_or(ExprEvalError::GetNoneFromContext)? + .try_to_owned() + .ok_or(ExprEvalError::OtherErr( + "cannot get `Object` from `BorrowObject`".to_string(), + ))?, + } } } else { element diff --git a/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval_pred.rs b/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval_pred.rs index 91dd6ad24b12..2be5703ae10c 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval_pred.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/utils/expr/eval_pred.rs @@ -261,25 +261,29 @@ impl EvalPred for Operand { if let Some(context) = _context { if let Some(elem) = context.get(tag.as_ref()) { if let Some(key) = prop_key { - if let Some(graph_element) = elem.as_graph_element() { - match key { - PropKey::Id => result = true, - PropKey::Label => { - result = graph_element.label().is_some(); - } - PropKey::Len => result = graph_element.len() > 0, - PropKey::All => { - // TODO(longbin) Do we need to look into the properties? - result = graph_element.details().is_some() - } - PropKey::Key(key) => { - if let Some(details) = graph_element.details() { - result = details.get_property(key).is_some(); + if let PropKey::Len = key { + result = elem.len() > 0 + } else { + if let Some(graph_element) = elem.as_graph_element() { + match key { + PropKey::Id => result = true, + PropKey::Label => { + result = graph_element.label().is_some(); + } + PropKey::Len => unreachable!(), + PropKey::All => { + // TODO(longbin) Do we need to look into the properties? + result = graph_element.details().is_some() + } + PropKey::Key(key) => { + if let Some(details) = graph_element.details() { + result = details.get_property(key).is_some(); + } } } + } else { + result = false } - } else { - result = false } } else { result = true; diff --git a/interactive_engine/executor/ir/integrated/tests/apply_test.rs b/interactive_engine/executor/ir/integrated/tests/apply_test.rs index bf76033f4b8e..66f16a381732 100644 --- a/interactive_engine/executor/ir/integrated/tests/apply_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/apply_test.rs @@ -29,7 +29,7 @@ mod test { use pegasus_server::job_pb as server_pb; use pegasus_server::JobRequest; use prost::Message; - use runtime::process::record::Entry; + use runtime::process::entry::Entry; use crate::common::test::*; @@ -230,9 +230,13 @@ mod test { Ok(res) => { let record = parse_result(res).unwrap(); if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { - if let Entry::OffGraph(cnt) = record.get(Some(TAG_A)).unwrap().as_ref() { - result_collection.push((vertex.id() as DefaultId, cnt.as_u64().unwrap())); - } + let cnt = record + .get(Some(TAG_A)) + .unwrap() + .as_object() + .unwrap(); + + result_collection.push((vertex.id() as DefaultId, cnt.as_u64().unwrap())); } } Err(e) => { @@ -273,13 +277,16 @@ mod test { Ok(res) => { let record = parse_result(res).unwrap(); if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { - if let Entry::OffGraph(cnt) = record.get(Some(TAG_A)).unwrap().as_ref() { - result_collection - .push((vertex.id() as DefaultId, Some(cnt.clone().as_u64().unwrap()))); - } else if let Entry::OffGraph(Object::None) = - record.get(Some(TAG_A)).unwrap().as_ref() - { + let object = record + .get(Some(TAG_A)) + .unwrap() + .as_object() + .unwrap(); + if object.eq(&Object::None) { result_collection.push((vertex.id() as DefaultId, None)); + } else { + result_collection + .push((vertex.id() as DefaultId, Some(object.as_u64().unwrap()))); } } } diff --git a/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs b/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs index 0299b537557e..104683c85e10 100644 --- a/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs @@ -31,6 +31,7 @@ mod test { use ir_common::NameOrId; use pegasus::api::{Map, Sink}; use pegasus::JobConf; + use runtime::process::entry::Entry; use runtime::process::operator::flatmap::FlatMapFuncGen; use runtime::process::operator::map::FilterMapFuncGen; use runtime::process::operator::source::SourceOperator; diff --git a/interactive_engine/executor/ir/integrated/tests/common/mod.rs b/interactive_engine/executor/ir/integrated/tests/common/mod.rs index b46a1119f8bf..01fa910b070e 100644 --- a/interactive_engine/executor/ir/integrated/tests/common/mod.rs +++ b/interactive_engine/executor/ir/integrated/tests/common/mod.rs @@ -23,7 +23,7 @@ pub mod test { use std::convert::{TryFrom, TryInto}; use std::sync::{Arc, Once}; - use graph_proxy::apis::{DynDetails, Edge, Vertex, VertexOrEdge, ID}; + use graph_proxy::apis::{DynDetails, Edge, Vertex, ID}; use ir_common::expr_parse::str_to_expr_pb; use ir_common::generated::algebra as pb; use ir_common::generated::common as common_pb; @@ -36,7 +36,8 @@ pub mod test { use pegasus_server::rpc::RpcSink; use pegasus_server::JobRequest; use prost::Message; - use runtime::process::record::{Entry, Record}; + use runtime::process::entry::DynEntry; + use runtime::process::record::Record; use runtime::IRJobAssembly; use runtime_integration::{InitializeJobAssembly, QueryExpGraph}; @@ -116,9 +117,9 @@ pub mod test { // append entry without moving head if let Some(tag) = tag { let columns = record.get_columns_mut(); - columns.insert(tag as usize, Arc::new(Entry::try_from(entry).unwrap())); + columns.insert(tag as usize, DynEntry::try_from(entry).unwrap()); } else { - record.append(Entry::try_from(entry).unwrap(), None); + record.append(DynEntry::try_from(entry).unwrap(), None); } } Some(record) diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index 7ea01f23adca..3f3264f217e7 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -19,7 +19,6 @@ mod common; #[cfg(test)] mod test { - use std::borrow::Borrow; use std::sync::Arc; use dyn_type::object; @@ -33,12 +32,14 @@ mod test { use pegasus::api::{Map, Sink}; use pegasus::result::ResultStream; use pegasus::JobConf; + use runtime::process::entry::Entry; use runtime::process::operator::flatmap::FlatMapFuncGen; - use runtime::process::operator::map::FilterMapFuncGen; + use runtime::process::operator::map::{FilterMapFuncGen, Intersection}; use runtime::process::operator::source::SourceOperator; - use runtime::process::record::{Entry, Record}; + use runtime::process::record::Record; use crate::common::test::*; + use pegasus_common::downcast::AsAny; // g.V() fn source_gen(alias: Option) -> Box + Send> { @@ -755,15 +756,20 @@ mod test { vec![expected_collection.clone(), expected_collection.clone(), expected_collection]; let mut result_collections: Vec> = vec![]; while let Some(Ok(record)) = result.next() { - if let Entry::Intersection(intersection) = record.get(Some(TAG_C)).unwrap().borrow() { - let mut result_collection: Vec = intersection - .clone() - .iter() - .map(|r| r.id() as usize) - .collect(); - result_collection.sort(); - result_collections.push(result_collection); - } + let intersection = record + .get(Some(TAG_C)) + .unwrap() + // .inner + .as_any_ref() + .downcast_ref::() + .unwrap(); + let mut result_collection: Vec = intersection + .clone() + .iter() + .map(|r| r.id() as usize) + .collect(); + result_collection.sort(); + result_collections.push(result_collection); } assert_eq!(result_collections, expected_collections) } @@ -828,15 +834,21 @@ mod test { let expected_collections = vec![vec![v4]]; let mut result_collections = vec![]; while let Some(Ok(record)) = result.next() { - if let Entry::Intersection(intersection) = record.get(Some(TAG_C)).unwrap().borrow() { - let mut result_collection: Vec = intersection - .clone() - .iter() - .map(|r| r.id() as DefaultId) - .collect(); - result_collection.sort(); - result_collections.push(result_collection); - } + let intersection = record + .get(Some(TAG_C)) + .unwrap() + // .inner + .as_any_ref() + .downcast_ref::() + .unwrap(); + + let mut result_collection: Vec = intersection + .clone() + .iter() + .map(|r| r.id() as DefaultId) + .collect(); + result_collection.sort(); + result_collections.push(result_collection); } assert_eq!(result_collections, expected_collections) } diff --git a/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs b/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs index dfa014b5438f..b2059a67625a 100644 --- a/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs @@ -28,7 +28,7 @@ mod test { use pegasus_client::builder::*; use pegasus_server::JobRequest; use prost::Message; - use runtime::process::record::Entry; + use runtime::process::entry::Entry; use crate::common::test::*; @@ -206,9 +206,8 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Entry::OffGraph(obj) = record.get(None).unwrap().as_ref() { - result_collection.push(obj.to_string()) - } + let obj = record.get(None).unwrap().as_object().unwrap(); + result_collection.push(obj.to_string()) } Err(e) => { panic!("err result {:?}", e); @@ -238,9 +237,8 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Entry::OffGraph(obj) = record.get(None).unwrap().as_ref() { - result_collection.push(obj.to_string()) - } + let obj = record.get(None).unwrap().as_object().unwrap(); + result_collection.push(obj.to_string()) } Err(e) => { panic!("err result {:?}", e); diff --git a/interactive_engine/executor/ir/integrated/tests/match_test.rs b/interactive_engine/executor/ir/integrated/tests/match_test.rs index dde3e4899484..33c40b82789f 100644 --- a/interactive_engine/executor/ir/integrated/tests/match_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/match_test.rs @@ -30,6 +30,7 @@ mod test { use ir_core::plan::physical::AsPhysical; use pegasus_client::builder::JobBuilder; use pegasus_server::JobRequest; + use runtime::process::entry::Entry; use crate::common::test::{ default_sink_target, initialize, parse_result, query_params, submit_query, CREATED_LABEL, diff --git a/interactive_engine/executor/ir/integrated/tests/pathxd_test.rs b/interactive_engine/executor/ir/integrated/tests/pathxd_test.rs index a6ab69b2ff52..6b1364ccad4a 100644 --- a/interactive_engine/executor/ir/integrated/tests/pathxd_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/pathxd_test.rs @@ -26,6 +26,7 @@ mod test { use pegasus_server::job_pb as server_pb; use pegasus_server::JobRequest; use prost::Message; + use runtime::process::entry::Entry; use crate::common::test::*; diff --git a/interactive_engine/executor/ir/integrated/tests/scan_test.rs b/interactive_engine/executor/ir/integrated/tests/scan_test.rs index bd90d4a651d6..d6a5b73e26c5 100644 --- a/interactive_engine/executor/ir/integrated/tests/scan_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/scan_test.rs @@ -26,6 +26,7 @@ mod test { use graph_store::common::DefaultId; use graph_store::ldbc::LDBCVertexParser; use ir_common::generated::algebra as pb; + use runtime::process::entry::Entry; use runtime::process::operator::source::SourceOperator; use runtime::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/error.rs b/interactive_engine/executor/ir/runtime/src/error.rs index c4571783419b..169a52456b8c 100644 --- a/interactive_engine/executor/ir/runtime/src/error.rs +++ b/interactive_engine/executor/ir/runtime/src/error.rs @@ -121,6 +121,8 @@ pub enum FnExecError { AccumError(String), /// Not supported error UnSupported(String), + /// Unreachable error + Unreachable, } impl FnExecError { @@ -151,6 +153,7 @@ impl std::fmt::Display for FnExecError { FnExecError::UnExpectedData(e) => write!(f, "Unexpected data type in exec {}", e), FnExecError::AccumError(e) => write!(f, "Accum error in exec {}", e), FnExecError::UnSupported(e) => write!(f, "Op not supported error in exec {}", e), + FnExecError::Unreachable => write!(f, "Unreachable error in exec"), } } } diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs new file mode 100644 index 000000000000..449098e1770a --- /dev/null +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -0,0 +1,636 @@ +// +//! Copyright 2022 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::any::Any; +use std::cmp::Ordering; +use std::convert::TryFrom; +use std::convert::TryInto; +use std::fmt::Debug; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +use dyn_type::{BorrowObject, Object}; +use graph_proxy::apis::{ + read_id, write_id, DynDetails, Edge, Element, GraphElement, GraphPath, Vertex, ID, +}; +use ir_common::error::ParsePbError; +use ir_common::generated::results as result_pb; +use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; + +use crate::process::operator::map::Intersection; + +#[derive(Debug)] +pub enum EntryDataType { + // TODO: A global id to denote current data; Currently, it is mostly vertex global id?? + Id, + V, + E, + P, + Obj, + Intersect, + Collection, +} + +impl PartialEq for EntryDataType { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (EntryDataType::Id, EntryDataType::Id) + | (EntryDataType::Id, EntryDataType::V) + | (EntryDataType::V, EntryDataType::Id) + | (EntryDataType::V, EntryDataType::V) + | (EntryDataType::E, EntryDataType::E) + | (EntryDataType::P, EntryDataType::P) + | (EntryDataType::Obj, EntryDataType::Obj) + | (EntryDataType::Intersect, EntryDataType::Intersect) + | (EntryDataType::Collection, EntryDataType::Collection) => true, + _ => false, + } + } +} + +pub trait Entry: Debug + Send + Sync + AsAny + Element { + fn get_type(&self) -> EntryDataType; + fn as_id(&self) -> Option; + fn as_graph_vertex(&self) -> Option<&Vertex> { + None + } + fn as_graph_edge(&self) -> Option<&Edge> { + None + } + fn as_graph_path(&self) -> Option<&GraphPath> { + None + } + fn as_object(&self) -> Option<&Object> { + None + } +} + +#[derive(Clone, Debug)] +pub struct DynEntry { + inner: Arc, +} + +impl AsAny for DynEntry { + fn as_any_mut(&mut self) -> &mut dyn Any { + // If you want to make self.inner as mutable,try Arc::get_mut(&mut self.inner) first. i.e., + // Arc::get_mut(&mut self.inner) + // .unwrap() + // .as_any_mut() + self + } + + fn as_any_ref(&self) -> &dyn Any { + self.inner.as_any_ref() + } +} + +impl DynEntry { + pub fn new(entry: E) -> Self { + DynEntry { inner: Arc::new(entry) } + } + + pub fn get_mut(&mut self) -> Option<&mut dyn Entry> { + Arc::get_mut(&mut self.inner) + } + + pub fn is_none(&self) -> bool { + match self.get_type() { + EntryDataType::Obj => self + .as_object() + .map(|obj| obj.eq(&Object::None)) + .unwrap_or(false), + _ => false, + } + } +} + +impl Entry for DynEntry { + fn get_type(&self) -> EntryDataType { + self.inner.get_type() + } + + fn as_id(&self) -> Option { + self.inner.as_id() + } + + fn as_graph_vertex(&self) -> Option<&Vertex> { + self.inner.as_graph_vertex() + } + + fn as_graph_edge(&self) -> Option<&Edge> { + self.inner.as_graph_edge() + } + + fn as_graph_path(&self) -> Option<&GraphPath> { + self.inner.as_graph_path() + } + + fn as_object(&self) -> Option<&Object> { + self.inner.as_object() + } +} + +impl Encode for DynEntry { + fn write_to(&self, writer: &mut W) -> std::io::Result<()> { + let entry_type = self.get_type(); + match entry_type { + EntryDataType::Id => { + writer.write_u8(0)?; + write_id(writer, self.as_id().unwrap())?; + } + EntryDataType::V => { + writer.write_u8(1)?; + self.as_graph_vertex() + .unwrap() + .write_to(writer)?; + } + EntryDataType::E => { + writer.write_u8(2)?; + self.as_graph_edge().unwrap().write_to(writer)?; + } + EntryDataType::P => { + writer.write_u8(3)?; + self.as_graph_path().unwrap().write_to(writer)?; + } + EntryDataType::Obj => { + writer.write_u8(4)?; + self.as_object().unwrap().write_to(writer)?; + } + EntryDataType::Intersect => { + writer.write_u8(5)?; + self.inner + .as_any_ref() + .downcast_ref::() + .unwrap() + .write_to(writer)?; + } + EntryDataType::Collection => { + writer.write_u8(6)?; + self.inner + .as_any_ref() + .downcast_ref::() + .unwrap() + .write_to(writer)?; + } + } + Ok(()) + } +} + +impl Decode for DynEntry { + fn read_from(reader: &mut R) -> std::io::Result { + let entry_type = reader.read_u8()?; + match entry_type { + 0 => { + let id = read_id(reader)?; + Ok(DynEntry::new(IDEntry { id })) + } + 1 => { + let vertex = Vertex::read_from(reader)?; + Ok(DynEntry::new(vertex)) + } + 2 => { + let edge = Edge::read_from(reader)?; + Ok(DynEntry::new(edge)) + } + 3 => { + let path = GraphPath::read_from(reader)?; + Ok(DynEntry::new(path)) + } + 4 => { + let obj = Object::read_from(reader)?; + Ok(DynEntry::new(obj)) + } + 5 => { + let intersect = Intersection::read_from(reader)?; + Ok(DynEntry::new(intersect)) + } + 6 => { + let collection = CollectionEntry::read_from(reader)?; + Ok(DynEntry::new(collection)) + } + _ => unreachable!(), + } + } +} + +impl Element for DynEntry { + fn as_graph_element(&self) -> Option<&dyn GraphElement> { + self.inner.as_graph_element() + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn as_borrow_object(&self) -> BorrowObject { + self.inner.as_borrow_object() + } +} + +impl GraphElement for DynEntry { + fn id(&self) -> ID { + match self.get_type() { + EntryDataType::Id => self.as_id().unwrap(), + EntryDataType::V => self.as_graph_vertex().unwrap().id(), + EntryDataType::E => self.as_graph_edge().unwrap().id(), + _ => unreachable!(), + } + } + + fn label(&self) -> Option { + match self.get_type() { + EntryDataType::Id => None, + EntryDataType::V => self + .as_graph_vertex() + .map(|v| v.label()) + .unwrap_or(None), + EntryDataType::E => self + .as_graph_edge() + .map(|v| v.label()) + .unwrap_or(None), + _ => unreachable!(), + } + } + + fn details(&self) -> Option<&DynDetails> { + match self.get_type() { + EntryDataType::Id => None, + EntryDataType::V => self + .as_graph_vertex() + .map(|v| v.details()) + .unwrap_or(None), + EntryDataType::E => self + .as_graph_edge() + .map(|v| v.details()) + .unwrap_or(None), + _ => unreachable!(), + } + } +} + +// demanded when need to key the entry +impl Hash for DynEntry { + fn hash(&self, state: &mut H) { + match self.get_type() { + EntryDataType::Id | EntryDataType::V => self.as_id().hash(state), + EntryDataType::E => self.as_graph_edge().hash(state), + EntryDataType::P => self.as_graph_path().hash(state), + EntryDataType::Obj => self.as_object().hash(state), + EntryDataType::Intersect => self + .as_any_ref() + .downcast_ref::() + .hash(state), + EntryDataType::Collection => self + .as_any_ref() + .downcast_ref::() + .hash(state), + } + } +} + +// demanded when need to key the entry; and order the entry; +impl PartialEq for DynEntry { + fn eq(&self, other: &Self) -> bool { + if (self.get_type()).eq(&other.get_type()) { + match self.get_type() { + EntryDataType::Id | EntryDataType::V => self.as_id().eq(&other.as_id()), + EntryDataType::E => self.as_graph_edge().eq(&other.as_graph_edge()), + EntryDataType::P => self.as_graph_path().eq(&other.as_graph_path()), + EntryDataType::Obj => self.as_object().eq(&other.as_object()), + EntryDataType::Intersect => self + .as_any_ref() + .downcast_ref::() + .eq(&other + .as_any_ref() + .downcast_ref::()), + EntryDataType::Collection => self + .as_any_ref() + .downcast_ref::() + .eq(&other + .as_any_ref() + .downcast_ref::()), + } + } else { + false + } + } +} + +// demanded when need to order the entry; +impl PartialOrd for DynEntry { + fn partial_cmp(&self, other: &Self) -> Option { + if (self.get_type()).eq(&other.get_type()) { + match self.get_type() { + EntryDataType::Id | EntryDataType::V => self.as_id().partial_cmp(&other.as_id()), + EntryDataType::E => self + .as_graph_edge() + .partial_cmp(&other.as_graph_edge()), + EntryDataType::P => self + .as_graph_path() + .partial_cmp(&other.as_graph_path()), + EntryDataType::Obj => self.as_object().partial_cmp(&other.as_object()), + EntryDataType::Intersect => self + .as_any_ref() + .downcast_ref::() + .partial_cmp( + &other + .as_any_ref() + .downcast_ref::(), + ), + EntryDataType::Collection => self + .as_any_ref() + .downcast_ref::() + .partial_cmp( + &other + .as_any_ref() + .downcast_ref::(), + ), + } + } else { + None + } + } +} + +// demanded when need to group (ToSet) the entry; +impl Eq for DynEntry {} + +// demanded when need to group (ToSum) the entry; +impl std::ops::Add for DynEntry { + type Output = DynEntry; + + fn add(self, rhs: Self) -> Self::Output { + if (self.get_type()).eq(&rhs.get_type()) { + if EntryDataType::Obj.eq(&self.get_type()) { + match (self.as_object(), rhs.as_object()) { + (Some(Object::Primitive(p1)), Some(Object::Primitive(p2))) => { + return DynEntry::new(Object::Primitive(p1.add(p2.clone()))); + } + _ => {} + } + } + } + DynEntry::new(Object::None) + } +} + +#[derive(Debug, Clone, Default)] +pub struct IDEntry { + id: ID, +} + +impl_as_any!(IDEntry); + +impl Entry for IDEntry { + fn get_type(&self) -> EntryDataType { + EntryDataType::Id + } + + fn as_id(&self) -> Option { + Some(self.id) + } +} + +impl Encode for IDEntry { + fn write_to(&self, writer: &mut W) -> std::io::Result<()> { + write_id(writer, self.id) + } +} + +impl Decode for IDEntry { + fn read_from(reader: &mut R) -> std::io::Result { + let id = read_id(reader)?; + Ok(IDEntry { id }) + } +} + +impl Element for IDEntry { + fn as_graph_element(&self) -> Option<&dyn GraphElement> { + Some(self) + } + + fn len(&self) -> usize { + 1 + } + + fn as_borrow_object(&self) -> BorrowObject { + self.id.into() + } +} + +impl GraphElement for IDEntry { + fn id(&self) -> ID { + self.id + } + + fn label(&self) -> Option { + None + } +} + +impl Entry for Vertex { + fn get_type(&self) -> EntryDataType { + EntryDataType::V + } + + fn as_id(&self) -> Option { + Some(self.id()) + } + + fn as_graph_vertex(&self) -> Option<&Vertex> { + Some(self) + } +} + +impl Entry for Edge { + fn get_type(&self) -> EntryDataType { + EntryDataType::E + } + + fn as_id(&self) -> Option { + Some(self.id()) + } + + fn as_graph_edge(&self) -> Option<&Edge> { + Some(self) + } +} + +impl Entry for Object { + fn get_type(&self) -> EntryDataType { + EntryDataType::Obj + } + + fn as_id(&self) -> Option { + None + } + + fn as_object(&self) -> Option<&Object> { + Some(self) + } +} + +impl Entry for Intersection { + fn get_type(&self) -> EntryDataType { + EntryDataType::Intersect + } + + fn as_id(&self) -> Option { + None + } +} + +impl Entry for GraphPath { + fn get_type(&self) -> EntryDataType { + EntryDataType::P + } + + fn as_id(&self) -> Option { + None + } + + fn as_graph_path(&self) -> Option<&GraphPath> { + Some(self) + } +} + +#[derive(Debug, Clone, Default, PartialEq, PartialOrd, Eq, Hash)] +pub struct CollectionEntry { + pub inner: Vec, +} + +impl_as_any!(CollectionEntry); + +impl Entry for CollectionEntry { + fn get_type(&self) -> EntryDataType { + EntryDataType::Collection + } + + fn as_id(&self) -> Option { + None + } +} + +impl Element for CollectionEntry { + fn as_graph_element(&self) -> Option<&dyn GraphElement> { + None + } + + fn len(&self) -> usize { + self.inner.len() + } + + fn as_borrow_object(&self) -> BorrowObject { + BorrowObject::None + } +} + +impl Encode for CollectionEntry { + fn write_to(&self, writer: &mut W) -> std::io::Result<()> { + self.inner.write_to(writer) + } +} + +impl Decode for CollectionEntry { + fn read_from(reader: &mut R) -> std::io::Result { + let inner = >::read_from(reader)?; + Ok(CollectionEntry { inner }) + } +} + +impl TryFrom for DynEntry { + type Error = ParsePbError; + fn try_from(e: result_pb::Element) -> Result { + if let Some(inner) = e.inner { + match inner { + result_pb::element::Inner::Vertex(v) => { + let vertex = Vertex::try_from(v)?; + Ok(DynEntry::new(vertex)) + } + result_pb::element::Inner::Edge(e) => { + let edge = Edge::try_from(e)?; + Ok(DynEntry::new(edge)) + } + result_pb::element::Inner::GraphPath(p) => { + let path = GraphPath::try_from(p)?; + Ok(DynEntry::new(path)) + } + result_pb::element::Inner::Object(o) => { + let obj = Object::try_from(o)?; + Ok(DynEntry::new(obj)) + } + } + } else { + Err(ParsePbError::EmptyFieldError("element inner is empty".to_string()))? + } + } +} + +impl TryFrom for DynEntry { + type Error = ParsePbError; + + fn try_from(entry_pb: result_pb::Entry) -> Result { + if let Some(inner) = entry_pb.inner { + match inner { + result_pb::entry::Inner::Element(e) => Ok(e.try_into()?), + result_pb::entry::Inner::Collection(c) => { + let collection = CollectionEntry { + inner: c + .collection + .into_iter() + .map(|e| e.try_into()) + .collect::, Self::Error>>()?, + }; + Ok(DynEntry::new(collection)) + } + } + } else { + Err(ParsePbError::EmptyFieldError("entry inner is empty".to_string()))? + } + } +} + +impl From for DynEntry { + fn from(v: Vertex) -> Self { + DynEntry::new(v) + } +} + +impl From for DynEntry { + fn from(e: Edge) -> Self { + DynEntry::new(e) + } +} + +impl From for DynEntry { + fn from(p: GraphPath) -> Self { + DynEntry::new(p) + } +} + +impl From for DynEntry { + fn from(o: Object) -> Self { + DynEntry::new(o) + } +} + +impl From for DynEntry { + fn from(c: CollectionEntry) -> Self { + DynEntry::new(c) + } +} diff --git a/interactive_engine/executor/ir/runtime/src/process/mod.rs b/interactive_engine/executor/ir/runtime/src/process/mod.rs index 4e89580286db..dfc57014a3d9 100644 --- a/interactive_engine/executor/ir/runtime/src/process/mod.rs +++ b/interactive_engine/executor/ir/runtime/src/process/mod.rs @@ -13,6 +13,7 @@ //! See the License for the specific language governing permissions and //! limitations under the License. +pub mod entry; pub mod functions; pub mod operator; pub mod record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs index 4b11e78d790d..d304c03287b1 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs @@ -25,24 +25,24 @@ use ir_common::KeyId; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use crate::error::{FnExecError, FnExecResult, FnGenError, FnGenResult}; +use crate::process::entry::{CollectionEntry, DynEntry, Entry}; use crate::process::operator::accum::accumulator::{ Accumulator, Count, DistinctCount, Maximum, Minimum, Sum, ToList, ToSet, }; use crate::process::operator::accum::AccumFactoryGen; use crate::process::operator::TagKey; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; #[derive(Debug, Clone)] pub enum EntryAccumulator { - // TODO(bingqing): more accum kind ToCount(Count<()>), - ToList(ToList), - ToMin(Minimum), - ToMax(Maximum), - ToSet(ToSet), - ToDistinctCount(DistinctCount), - ToSum(Sum), - ToAvg(Sum, Count<()>), + ToList(ToList), + ToMin(Minimum), + ToMax(Maximum), + ToSet(ToSet), + ToDistinctCount(DistinctCount), + ToSum(Sum), + ToAvg(Sum, Count<()>), } /// Accumulator for Record, including multiple accumulators for entries(columns) in Record. @@ -56,7 +56,7 @@ pub struct RecordAccumulator { impl Accumulator for RecordAccumulator { fn accum(&mut self, mut next: Record) -> FnExecResult<()> { for (accumulator, tag_key, _) in self.accum_ops.iter_mut() { - let entry = tag_key.get_entry(&mut next)?; + let entry = tag_key.get_arc_entry(&mut next)?; accumulator.accum(entry)?; } Ok(()) @@ -66,14 +66,14 @@ impl Accumulator for RecordAccumulator { let mut record = Record::default(); for (accumulator, _, alias) in self.accum_ops.iter_mut() { let entry = accumulator.finalize()?; - record.append(entry, alias.clone()); + record.append_arc_entry(entry, alias.clone()); } Ok(record) } } -impl Accumulator for EntryAccumulator { - fn accum(&mut self, next: Entry) -> FnExecResult<()> { +impl Accumulator for EntryAccumulator { + fn accum(&mut self, next: DynEntry) -> FnExecResult<()> { // ignore non-exist tag/label/property values; if !next.is_none() { match self { @@ -94,15 +94,15 @@ impl Accumulator for EntryAccumulator { } } - fn finalize(&mut self) -> FnExecResult { + fn finalize(&mut self) -> FnExecResult { match self { EntryAccumulator::ToCount(count) => { let cnt = count.finalize()?; - Ok(object!(cnt).into()) + Ok(DynEntry::new(object!(cnt))) } EntryAccumulator::ToList(list) => { - let list_entry = list.finalize()?; - Ok(Entry::Collection(list_entry)) + let list_entry: Vec = list.finalize()?; + Ok(DynEntry::new(CollectionEntry { inner: list_entry })) } EntryAccumulator::ToMin(min) => min .finalize()? @@ -112,11 +112,11 @@ impl Accumulator for EntryAccumulator { .ok_or(FnExecError::accum_error("max_entry is none")), EntryAccumulator::ToSet(set) => { let set_entry = set.finalize()?; - Ok(Entry::Collection(set_entry)) + Ok(DynEntry::new(CollectionEntry { inner: set_entry })) } EntryAccumulator::ToDistinctCount(distinct_count) => { let cnt = distinct_count.finalize()?; - Ok(object!(cnt).into()) + Ok(DynEntry::new(object!(cnt))) } EntryAccumulator::ToSum(sum) => sum .finalize()? @@ -127,7 +127,7 @@ impl Accumulator for EntryAccumulator { .ok_or(FnExecError::accum_error("sum_entry is none"))?; let cnt = count.finalize()?; // TODO: confirm if it should be Object::None, or throw error; - let result = (Object::None).into(); + let result = DynEntry::new(Object::None); if cnt == 0 { warn!("cnt value is 0 in accum avg"); Ok(result) @@ -144,7 +144,7 @@ impl Accumulator for EntryAccumulator { .map(|val| val.div(primitive_cnt)) .map_err(|e| FnExecError::accum_error(&format!("{}", e)))?; let result_obj: Object = result.into(); - Ok(result_obj.into()) + Ok(DynEntry::new(result_obj)) } } } else { @@ -256,31 +256,31 @@ impl Decode for EntryAccumulator { Ok(EntryAccumulator::ToCount(cnt)) } 1 => { - let list = >::read_from(reader)?; + let list = >::read_from(reader)?; Ok(EntryAccumulator::ToList(list)) } 2 => { - let min = >::read_from(reader)?; + let min = >::read_from(reader)?; Ok(EntryAccumulator::ToMin(min)) } 3 => { - let max = >::read_from(reader)?; + let max = >::read_from(reader)?; Ok(EntryAccumulator::ToMax(max)) } 4 => { - let set = >::read_from(reader)?; + let set = >::read_from(reader)?; Ok(EntryAccumulator::ToSet(set)) } 5 => { - let distinct_count = >::read_from(reader)?; + let distinct_count = >::read_from(reader)?; Ok(EntryAccumulator::ToDistinctCount(distinct_count)) } 6 => { - let sum = >::read_from(reader)?; + let sum = >::read_from(reader)?; Ok(EntryAccumulator::ToSum(sum)) } 7 => { - let sum = >::read_from(reader)?; + let sum = >::read_from(reader)?; let count = >::read_from(reader)?; Ok(EntryAccumulator::ToAvg(sum, count)) } @@ -318,7 +318,6 @@ impl Decode for RecordAccumulator { #[cfg(test)] mod tests { - use std::borrow::BorrowMut; use std::cmp::Ordering; use ir_common::generated::algebra as pb; @@ -327,10 +326,12 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::{CollectionEntry, DynEntry, Entry}; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::accum::AccumFactoryGen; use crate::process::operator::tests::{init_source, init_vertex1, init_vertex2, TAG_A, TAG_B}; - use crate::process::record::{Entry, Record}; + use crate::process::record::Record; + use pegasus_common::downcast::AsAny; fn fold_test(source: Vec, fold_opr_pb: pb::GroupBy) -> ResultStream { let conf = JobConf::new("fold_test"); @@ -367,11 +368,12 @@ mod tests { }; let fold_opr_pb = pb::GroupBy { mappings: vec![], functions: vec![function] }; let mut result = fold_test(init_source(), fold_opr_pb); - let mut fold_result = Entry::Collection(vec![]); - let expected_result = Entry::Collection(vec![init_vertex1().into(), init_vertex2().into()]); + let mut fold_result = CollectionEntry::default().into(); + let expected_result = + CollectionEntry { inner: vec![init_vertex1().into(), init_vertex2().into()] }.into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - fold_result = entry.as_ref().clone(); + fold_result = entry.clone(); } } assert_eq!(fold_result, expected_result); @@ -387,11 +389,12 @@ mod tests { }; let fold_opr_pb = pb::GroupBy { mappings: vec![], functions: vec![function] }; let mut result = fold_test(init_source(), fold_opr_pb); - let mut fold_result = Entry::Collection(vec![]); - let expected_result = Entry::Collection(vec![init_vertex1().into(), init_vertex2().into()]); + let mut fold_result = CollectionEntry::default().into(); + let expected_result = + CollectionEntry { inner: vec![init_vertex1().into(), init_vertex2().into()] }.into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(None) { - fold_result = entry.as_ref().clone(); + fold_result = entry.clone(); } } assert_eq!(fold_result, expected_result); @@ -410,12 +413,7 @@ mod tests { let mut cnt = 0; if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - cnt = match entry.as_ref() { - Entry::OffGraph(cnt) => cnt.as_u64().unwrap(), - _ => { - unreachable!() - } - }; + cnt = entry.as_object().unwrap().as_u64().unwrap(); } } assert_eq!(cnt, 2); @@ -436,20 +434,14 @@ mod tests { }; let fold_opr_pb = pb::GroupBy { mappings: vec![], functions: vec![function_1, function_2] }; let mut result = fold_test(init_source(), fold_opr_pb); - let mut fold_result: (Entry, Entry) = (Entry::Collection(vec![]), object!(0).into()); - let expected_result: (Entry, Entry) = - (Entry::Collection(vec![init_vertex1().into(), init_vertex2().into()]), object!(2).into()); + let mut fold_result: (DynEntry, DynEntry) = (CollectionEntry::default().into(), object!(0).into()); + let expected_result: (DynEntry, DynEntry) = ( + CollectionEntry { inner: vec![init_vertex1().into(), init_vertex2().into()] }.into(), + object!(2).into(), + ); if let Some(Ok(record)) = result.next() { - let collection_entry = record - .get(Some(TAG_A)) - .unwrap() - .as_ref() - .clone(); - let count_entry = record - .get(Some(TAG_B)) - .unwrap() - .as_ref() - .clone(); + let collection_entry = record.get(Some(TAG_A)).unwrap().clone(); + let count_entry = record.get(Some(TAG_B)).unwrap().clone(); fold_result = (collection_entry, count_entry); } assert_eq!(fold_result, expected_result); @@ -470,13 +462,7 @@ mod tests { let mut res = 0.into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - res = match entry.as_ref() { - // this is Prop, since get_entry returns entry type of prop - Entry::OffGraph(obj) => obj.clone(), - _ => { - unreachable!() - } - }; + res = entry.as_object().unwrap().clone(); } } assert_eq!(res, object!(27)); @@ -497,13 +483,7 @@ mod tests { let mut res = "".into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - res = match entry.as_ref() { - // this is Prop, since get_entry returns entry type of prop - Entry::OffGraph(obj) => obj.clone(), - _ => { - unreachable!() - } - }; + res = entry.as_object().unwrap().clone(); } } assert_eq!(res, object!("vadas")); @@ -529,12 +509,7 @@ mod tests { let mut cnt = 0; if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - cnt = match entry.as_ref() { - Entry::OffGraph(cnt) => cnt.as_u64().unwrap(), - _ => { - unreachable!() - } - }; + cnt = entry.as_object().unwrap().as_u64().unwrap(); } } assert_eq!(cnt, 2); @@ -554,16 +529,22 @@ mod tests { }; let fold_opr_pb = pb::GroupBy { mappings: vec![], functions: vec![function] }; let mut result = fold_test(source, fold_opr_pb); - let mut fold_result = Entry::Collection(vec![]); - let expected_result = Entry::Collection(vec![init_vertex1().into(), init_vertex2().into()]); + let mut fold_result = CollectionEntry::default(); + let expected_result = CollectionEntry { inner: vec![init_vertex1().into(), init_vertex2().into()] }; if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - fold_result = entry.as_ref().clone(); + fold_result = entry + // .inner + .as_any_ref() + .downcast_ref::() + .unwrap() + .clone(); } } - if let Entry::Collection(collection) = fold_result.borrow_mut() { - collection.sort_by(|v1, v2| v1.partial_cmp(&v2).unwrap_or(Ordering::Equal)); - } + fold_result + .inner + .sort_by(|v1, v2| v1.partial_cmp(&v2).unwrap_or(Ordering::Equal)); + assert_eq!(fold_result, expected_result); } @@ -583,12 +564,7 @@ mod tests { let mut res = "".into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - res = match entry.as_ref() { - Entry::OffGraph(obj) => obj.clone(), - _ => { - unreachable!() - } - }; + res = entry.as_object().unwrap().clone(); } } assert_eq!(res, object!(60)); @@ -610,12 +586,7 @@ mod tests { let mut res = "".into(); if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(None) { - res = match entry.as_ref() { - Entry::OffGraph(obj) => obj.clone(), - _ => { - unreachable!() - } - }; + res = entry.as_object().unwrap().clone(); } } assert_eq!(res, object!(20)); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs b/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs index acd46d7f843a..6865cc883d32 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs @@ -62,6 +62,7 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::operator::filter::FilterFuncGen; use crate::process::operator::tests::{init_source, PERSON_LABEL}; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs index 04c454c9371c..ccccb95382f7 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs @@ -24,64 +24,74 @@ use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenError, FnGenResult}; +use crate::process::entry::{Entry, EntryDataType}; use crate::process::operator::flatmap::FlatMapFuncGen; -use crate::process::record::{Entry, Record, RecordExpandIter, RecordPathExpandIter}; +use crate::process::record::{Record, RecordExpandIter, RecordPathExpandIter}; -pub struct EdgeExpandOperator> { +pub struct EdgeExpandOperator { start_v_tag: Option, alias: Option, stmt: Box>, expand_opt: ExpandOpt, } -impl + 'static> FlatMapFunction for EdgeExpandOperator { +impl FlatMapFunction for EdgeExpandOperator { type Target = DynIter; fn exec(&self, mut input: Record) -> FnResult { if let Some(entry) = input.get(self.start_v_tag) { - if let Some(v) = entry.as_graph_vertex() { - let id = v.id(); - let iter = self.stmt.exec(id)?; - match self.expand_opt { - ExpandOpt::Vertex => { - let neighbors_iter = iter.map(|e| match e.into() { - Entry::E(e) => Vertex::new( - e.get_other_id(), - e.get_other_label().cloned(), - DynDetails::default(), - ), - _ => { - unreachable!() - } - }); - Ok(Box::new(RecordExpandIter::new( - input, - self.alias.as_ref(), - Box::new(neighbors_iter), - ))) - } - ExpandOpt::Edge => { - Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) - } - ExpandOpt::Degree => { - let degree = iter.count(); - input.append(object!(degree), self.alias); - Ok(Box::new(vec![input].into_iter())) + match entry.get_type() { + EntryDataType::Id | EntryDataType::V => { + let id = entry.as_id().ok_or(FnExecError::Unreachable)?; + let iter = self.stmt.exec(id)?; + match self.expand_opt { + // the case of expand edge, and get end vertex; + ExpandOpt::Vertex => { + let neighbors_iter = iter.map(|e| { + if let Some(e) = e.as_graph_edge() { + Vertex::new( + e.get_other_id(), + e.get_other_label().cloned(), + DynDetails::default(), + ) + } else { + unreachable!() + } + }); + Ok(Box::new(RecordExpandIter::new( + input, + self.alias.as_ref(), + Box::new(neighbors_iter), + ))) + } + // the case of expand neighbors, including edges/vertices + ExpandOpt::Edge => { + Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) + } + // the case of get degree. TODO: this case should be a `Map` + ExpandOpt::Degree => { + let degree = iter.count(); + input.append(object!(degree), self.alias); + Ok(Box::new(vec![input].into_iter())) + } } } - } else if let Some(graph_path) = entry.as_graph_path() { - let path_end = graph_path - .get_path_end() - .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; - let id = path_end.id(); - let iter = self.stmt.exec(id)?; - let curr_path = graph_path.clone(); - Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) - } else { - Err(FnExecError::unexpected_data_error(&format!( + EntryDataType::P => { + let graph_path = entry + .as_graph_path() + .ok_or(FnExecError::Unreachable)?; + let path_end = graph_path + .get_path_end() + .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; + let id = path_end.id(); + let iter = self.stmt.exec(id)?; + let curr_path = graph_path.clone(); + Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) + } + _ => Err(FnExecError::unexpected_data_error(&format!( "Cannot Expand from current entry {:?}", entry - )))? + )))?, } } else { Ok(Box::new(vec![].into_iter())) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs index 6c5e881c7063..7816770ced35 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs @@ -23,6 +23,7 @@ use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; +use crate::process::entry::Entry; use crate::process::operator::flatmap::FlatMapFuncGen; use crate::process::record::{Record, RecordExpandIter}; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs index 8029efa10735..2947435375b0 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs @@ -14,15 +14,17 @@ //! limitations under the License. use std::convert::TryInto; -use std::sync::Arc; +use graph_proxy::apis::Element; use ir_common::generated::algebra as algebra_pb; use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; +use crate::process::entry::{CollectionEntry, EntryDataType}; use crate::process::operator::flatmap::FlatMapFuncGen; -use crate::process::record::{Entry, Record}; +use crate::process::operator::map::Intersection; +use crate::process::record::Record; #[derive(Debug)] /// Unfold the Collection entry referred by a given `tag`. @@ -48,33 +50,38 @@ impl FlatMapFunction for UnfoldOperator { // take head in case that head entry is an arc clone of `self.tag`; // besides, head will be replaced by the items in collections anyway. input.take(None); - if let Some(entry) = Arc::get_mut(&mut entry) { - match entry { - Entry::Collection(collection) => { + if let Some(entry) = entry.get_mut() { + match entry.get_type() { + EntryDataType::Collection => { + let collection = entry + .as_any_mut() + .downcast_mut::() + .unwrap(); let mut res = Vec::with_capacity(collection.len()); - for item in collection.drain(..) { + for item in collection.inner.drain(..) { let mut new_entry = input.clone(); new_entry.append(item, self.alias); res.push(new_entry); } Ok(Box::new(res.into_iter())) } - Entry::Intersection(intersection) => { + EntryDataType::Intersect => { + let intersection = entry + .as_any_mut() + .downcast_mut::() + .unwrap(); let mut res = Vec::with_capacity(intersection.len()); - for item in intersection.iter() { + for item in intersection.drain() { let mut new_entry = input.clone(); - new_entry.append(item.clone(), self.alias); + new_entry.append(item, self.alias); res.push(new_entry); } Ok(Box::new(res.into_iter())) } - Entry::P(_) => { - // TODO: to support path_unwinding - Err(FnExecError::unsupported_error(&format!( - "unfold path entry {:?} in UnfoldOperator", - entry - )))? - } + EntryDataType::P => Err(FnExecError::unsupported_error(&format!( + "unfold path entry {:?} in UnfoldOperator", + entry + )))?, _ => Err(FnExecError::unexpected_data_error(&format!( "unfold entry {:?} in UnfoldOperator", entry @@ -116,6 +123,7 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::functions::FoldGen; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::flatmap::FlatMapFuncGen; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/group/fold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/group/fold.rs index e740daab508e..0537ea31ddd5 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/group/fold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/group/fold.rs @@ -23,7 +23,7 @@ use pegasus_server::job_pb as server_pb; use crate::error::{FnGenError, FnGenResult}; use crate::process::functions::FoldGen; use crate::process::operator::accum::{AccumFactoryGen, RecordAccumulator}; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; impl FoldGen for algebra_pb::GroupBy { fn get_accum_kind(&self) -> server_pb::AccumKind { @@ -65,7 +65,7 @@ struct CountAlias { impl MapFunction for CountAlias { fn exec(&self, cnt: u64) -> FnResult { - let cnt_entry: Entry = object!(cnt).into(); + let cnt_entry = object!(cnt); Ok(Record::new(cnt_entry, self.alias.clone())) } } @@ -79,9 +79,10 @@ mod tests { use pegasus::JobConf; use pegasus_server::job_pb as server_pb; + use crate::process::entry::Entry; use crate::process::functions::FoldGen; use crate::process::operator::tests::{init_source, TAG_A}; - use crate::process::record::{Entry, Record}; + use crate::process::record::Record; fn count_test(source: Vec, fold_opr_pb: pb::GroupBy) -> ResultStream { let conf = JobConf::new("fold_test"); @@ -118,12 +119,7 @@ mod tests { let mut cnt = 0; if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(None) { - cnt = match entry.as_ref() { - Entry::OffGraph(cnt) => cnt.as_u64().unwrap(), - _ => { - unreachable!() - } - }; + cnt = entry.as_object().unwrap().as_u64().unwrap(); } } assert_eq!(cnt, 2); @@ -142,12 +138,7 @@ mod tests { let mut cnt = 0; if let Some(Ok(record)) = result.next() { if let Some(entry) = record.get(Some(TAG_A)) { - cnt = match entry.as_ref() { - Entry::OffGraph(cnt) => cnt.as_u64().unwrap(), - _ => { - unreachable!() - } - }; + cnt = entry.as_object().unwrap().as_u64().unwrap(); } } assert_eq!(cnt, 2); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/group/group.rs b/interactive_engine/executor/ir/runtime/src/process/operator/group/group.rs index 043a40a3e9ce..d994d08627b3 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/group/group.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/group/group.rs @@ -96,12 +96,13 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::{CollectionEntry, DynEntry, Entry}; use crate::process::functions::GroupGen; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::tests::{ init_source, init_vertex1, init_vertex2, PERSON_LABEL, TAG_A, TAG_B, TAG_C, }; - use crate::process::record::{Entry, Record}; + use crate::process::record::Record; // v1: marko, 29; // v2: vadas, 27; @@ -166,18 +167,18 @@ mod tests { let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = [ - (init_vertex1().into(), Entry::Collection(vec![(init_vertex1().into())])), - (init_vertex2().into(), Entry::Collection(vec![(init_vertex2().into())])), - (init_vertex3().into(), Entry::Collection(vec![(init_vertex3().into())])), + let expected_result: HashSet<(DynEntry, DynEntry)> = [ + (init_vertex1().into(), CollectionEntry { inner: vec![init_vertex1().into()] }.into()), + (init_vertex2().into(), CollectionEntry { inner: vec![init_vertex2().into()] }.into()), + (init_vertex3().into(), CollectionEntry { inner: vec![init_vertex3().into()] }.into()), ] .iter() .cloned() .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_A)).unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), val.clone())); } assert_eq!(group_result, expected_result); @@ -197,23 +198,30 @@ mod tests { }; let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); - let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = [ + let mut group_result = vec![]; + let mut expected_result: Vec<(Object, DynEntry)> = vec![ ( - object!("marko").into(), - Entry::Collection(vec![(init_vertex1().into()), (init_vertex3().into())]), + object!("marko"), + DynEntry::new(CollectionEntry { + inner: vec![DynEntry::new(init_vertex1()), DynEntry::new(init_vertex3())], + }), ), - (object!("vadas").into(), Entry::Collection(vec![(init_vertex2().into())])), - ] - .iter() - .cloned() - .collect(); - + ( + object!("vadas"), + DynEntry::new(CollectionEntry { inner: vec![DynEntry::new(init_vertex2())] }), + ), + ]; while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); - group_result.insert((key.clone(), val.clone())); + let key = result + .get(Some(TAG_A)) + .unwrap() + .as_object() + .unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); + group_result.push((key.clone(), val.clone())); } + expected_result.sort_by(|o1, o2| o1.0.cmp(&o2.0)); + group_result.sort_by(|o1, o2| o1.0.cmp(&o2.0)); assert_eq!(group_result, expected_result); } @@ -237,18 +245,18 @@ mod tests { pb::GroupBy { mappings: vec![key_alias_1, key_alias_2], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<((Entry, Entry), Entry)> = [ + let expected_result: HashSet<((DynEntry, DynEntry), DynEntry)> = [ ( (object!(1).into(), object!("marko").into()), - Entry::Collection(vec![(init_vertex1().into())]), + CollectionEntry { inner: vec![init_vertex1().into()] }.into(), ), ( (object!(2).into(), object!("vadas").into()), - Entry::Collection(vec![(init_vertex2().into())]), + CollectionEntry { inner: vec![init_vertex2().into()] }.into(), ), ( (object!(3).into(), object!("marko").into()), - Entry::Collection(vec![(init_vertex3().into())]), + CollectionEntry { inner: vec![init_vertex3().into()] }.into(), ), ] .iter() @@ -256,9 +264,9 @@ mod tests { .collect(); while let Some(Ok(result)) = result.next() { - let key_1 = result.get(Some(TAG_A)).unwrap().as_ref(); - let key_2 = result.get(Some(TAG_B)).unwrap().as_ref(); - let val = result.get(Some(TAG_C)).unwrap().as_ref(); + let key_1 = result.get(Some(TAG_A)).unwrap(); + let key_2 = result.get(Some(TAG_B)).unwrap(); + let val = result.get(Some(TAG_C)).unwrap(); group_result.insert(((key_1.clone(), key_2.clone()), val.clone())); } assert_eq!(group_result, expected_result); @@ -285,18 +293,18 @@ mod tests { pb::GroupBy { mappings: vec![key_alias], functions: vec![function_1, function_2] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, (Entry, Entry))> = [ + let expected_result: HashSet<(DynEntry, (DynEntry, DynEntry))> = [ ( init_vertex1().into(), - (Entry::Collection(vec![(init_vertex1().into())]), object!(1u64).into()), + (CollectionEntry { inner: vec![init_vertex1().into()] }.into(), object!(1u64).into()), ), ( init_vertex2().into(), - (Entry::Collection(vec![(init_vertex2().into())]), object!(1u64).into()), + (CollectionEntry { inner: vec![init_vertex2().into()] }.into(), object!(1u64).into()), ), ( init_vertex3().into(), - (Entry::Collection(vec![(init_vertex3().into())]), object!(1u64).into()), + (CollectionEntry { inner: vec![init_vertex3().into()] }.into(), object!(1u64).into()), ), ] .iter() @@ -304,9 +312,9 @@ mod tests { .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_C)).unwrap().as_ref(); - let val_1 = result.get(Some(TAG_A)).unwrap().as_ref(); - let val_2 = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_C)).unwrap(); + let val_1 = result.get(Some(TAG_A)).unwrap(); + let val_2 = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), (val_1.clone(), val_2.clone()))); } assert_eq!(group_result, expected_result); @@ -327,7 +335,7 @@ mod tests { let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = [ + let expected_result: HashSet<(DynEntry, DynEntry)> = [ (init_vertex1().into(), object!(1u64).into()), (init_vertex2().into(), object!(1u64).into()), (init_vertex3().into(), object!(1u64).into()), @@ -336,8 +344,8 @@ mod tests { .cloned() .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_A)).unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), val.clone())); } assert_eq!(group_result, expected_result); @@ -358,7 +366,7 @@ mod tests { let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = [ + let expected_result: HashSet<(DynEntry, DynEntry)> = [ (object!("marko").into(), object!(2u64).into()), (object!("vadas").into(), object!(1u64).into()), ] @@ -366,8 +374,8 @@ mod tests { .cloned() .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_A)).unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), val.clone())); } assert_eq!(group_result, expected_result); @@ -388,15 +396,15 @@ mod tests { let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = + let expected_result: HashSet<(DynEntry, DynEntry)> = [(object!("vadas").into(), object!(27).into()), (object!("marko").into(), object!(27).into())] .iter() .cloned() .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_A)).unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), val.clone())); } assert_eq!(group_result, expected_result); @@ -417,15 +425,15 @@ mod tests { let group_opr_pb = pb::GroupBy { mappings: vec![key_alias], functions: vec![function] }; let mut result = group_test(group_opr_pb); let mut group_result = HashSet::new(); - let expected_result: HashSet<(Entry, Entry)> = + let expected_result: HashSet<(DynEntry, DynEntry)> = [(object!("marko").into(), object!(29).into()), (object!("vadas").into(), object!(27).into())] .iter() .cloned() .collect(); while let Some(Ok(result)) = result.next() { - let key = result.get(Some(TAG_A)).unwrap().as_ref(); - let val = result.get(Some(TAG_B)).unwrap().as_ref(); + let key = result.get(Some(TAG_A)).unwrap(); + let val = result.get(Some(TAG_B)).unwrap(); group_result.insert((key.clone(), val.clone())); } assert_eq!(group_result, expected_result); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs b/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs index 334536b52746..30dedf558838 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs @@ -49,13 +49,15 @@ impl JoinKeyGen for algebra_pb::Join { #[cfg(test)] mod tests { - use graph_proxy::apis::{DynDetails, GraphElement, Vertex, ID}; + use graph_proxy::apis::GraphElement; + use graph_proxy::apis::{DynDetails, Vertex, ID}; use ir_common::generated::algebra as pb; use ir_common::generated::algebra::join::JoinKind; use ir_common::generated::common as common_pb; use pegasus::api::{Join, KeyBy, Map, PartitionByKey, Sink}; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::functions::JoinKeyGen; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs b/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs index 889b6d5203b2..618303f79467 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs @@ -81,13 +81,15 @@ impl KeyFunctionGen for algebra_pb::Dedup { mod tests { use ahash::HashMap; use dyn_type::Object; - use graph_proxy::apis::{DynDetails, GraphElement, Vertex, ID}; + use graph_proxy::apis::GraphElement; + use graph_proxy::apis::{DynDetails, Vertex, ID}; use ir_common::generated::algebra as pb; use ir_common::generated::common as common_pb; use ir_common::NameOrId; use pegasus::api::{Dedup, KeyBy, Map, Sink}; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::operator::keyed::KeyFunctionGen; use crate::process::operator::tests::PERSON_LABEL; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs index 89f868313a67..ada7ddc9d421 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs @@ -21,8 +21,9 @@ use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; +use crate::process::entry::{DynEntry, Entry, EntryDataType}; use crate::process::operator::map::FilterMapFuncGen; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; /// An Auxilia operator to get extra information for the current entity. /// Specifically, we will update the old entity by appending the new extra information, @@ -38,20 +39,23 @@ struct AuxiliaOperator { impl FilterMapFunction for AuxiliaOperator { fn exec(&self, mut input: Record) -> FnResult> { if let Some(entry) = input.get(self.tag) { - let entry = entry.clone(); - // Make sure there is anything to query with // Note that we need to guarantee the requested column if it has any alias, // e.g., for g.V().out().as("a").has("name", "marko"), we should compile as: // g.V().out().auxilia(as("a"))... where we give alias in auxilia, // then we set tag=None and alias="a" in auxilia - // TODO: it seems that we do not really care about getting head from curr or "a", we only need to save the updated entry with expected alias "a" - if self.query_params.is_queryable() { - // If queryable, then turn into graph element and do the query - let graph = get_graph().ok_or(FnExecError::NullGraphError)?; - let new_entry: Option = if let Some(v) = entry.as_graph_vertex() { - let mut result_iter = graph.get_vertex(&[v.id()], &self.query_params)?; + // If queryable, then turn into graph element and do the query + let graph = get_graph().ok_or(FnExecError::NullGraphError)?; + let new_entry: Option = match entry.get_type() { + EntryDataType::Id | EntryDataType::V => { + let id = entry.as_id().unwrap(); + let mut result_iter = graph.get_vertex(&[id], &self.query_params)?; result_iter.next().map(|mut vertex| { - if let Some(details) = v.details() { + // TODO:confirm the update case, and avoid it if possible. + if let Some(details) = entry + .as_graph_vertex() + .map(|v| v.details()) + .unwrap_or(None) + { if let Some(properties) = details.get_all_properties() { for (key, val) in properties { vertex @@ -60,35 +64,36 @@ impl FilterMapFunction for AuxiliaOperator { } } } - vertex.into() + DynEntry::new(vertex) }) - } else if let Some(e) = entry.as_graph_edge() { - let mut result_iter = graph.get_edge(&[e.id()], &self.query_params)?; + } + EntryDataType::E => { + let id = entry.as_id().unwrap(); + let mut result_iter = graph.get_edge(&[id], &self.query_params)?; result_iter.next().map(|mut edge| { - if let Some(details) = e.details() { + if let Some(details) = entry + .as_graph_edge() + .map(|e| e.details()) + .unwrap_or(None) + { if let Some(properties) = details.get_all_properties() { for (key, val) in properties { edge.get_details_mut().insert_property(key, val); } } } - edge.into() + DynEntry::new(edge) }) - } else { - Err(FnExecError::unexpected_data_error(&format!( - "neither Vertex nor Edge entry is accessed in `Auxilia` operator, the entry is {:?}", - entry - )))? - }; - if new_entry.is_some() { - input.append(new_entry.unwrap(), self.alias.clone()); - } else { - return Ok(None); } + _ => Err(FnExecError::unexpected_data_error(&format!( + "neither Vertex nor Edge entry is accessed in `Auxilia` operator, the entry is {:?}", + entry + )))?, + }; + if new_entry.is_some() { + input.append(new_entry.unwrap(), self.alias.clone()); } else { - if self.alias.is_some() { - input.append_arc_entry(entry, self.alias.clone()); - } + return Ok(None); } for remove_tag in &self.remove_tags { @@ -102,13 +107,35 @@ impl FilterMapFunction for AuxiliaOperator { } } +#[derive(Debug)] +struct SimpleAuxiliaOperator { + tag: Option, + alias: Option, + remove_tags: Vec, +} + +impl FilterMapFunction for SimpleAuxiliaOperator { + fn exec(&self, mut input: Record) -> FnResult> { + if input.get(self.tag).is_none() { + return Ok(None); + } + if self.alias.is_some() { + input.append_arc_entry(input.get(self.tag).unwrap().clone(), self.alias.clone()); + } + for remove_tag in &self.remove_tags { + input.take(Some(remove_tag)); + } + Ok(Some(input)) + } +} + impl FilterMapFuncGen for algebra_pb::Auxilia { fn gen_filter_map(self) -> FnGenResult>> { let tag = self .tag .map(|alias| alias.try_into()) .transpose()?; - let query_params = self.params.try_into()?; + let query_params: QueryParams = self.params.try_into()?; let alias = self .alias .map(|alias| alias.try_into()) @@ -118,10 +145,18 @@ impl FilterMapFuncGen for algebra_pb::Auxilia { .into_iter() .map(|alias| alias.try_into()) .collect::>()?; - let auxilia_operator = AuxiliaOperator { tag, query_params, alias, remove_tags }; - if log_enabled!(log::Level::Debug) && pegasus::get_current_worker().index == 0 { - debug!("Runtime AuxiliaOperator: {:?}", auxilia_operator); + if query_params.is_queryable() { + let auxilia_operator = AuxiliaOperator { tag, query_params, alias, remove_tags }; + if log_enabled!(log::Level::Debug) && pegasus::get_current_worker().index == 0 { + debug!("Runtime AuxiliaOperator: {:?}", auxilia_operator); + } + Ok(Box::new(auxilia_operator)) + } else { + let auxilia_operator = SimpleAuxiliaOperator { tag, alias, remove_tags }; + if log_enabled!(log::Level::Debug) && pegasus::get_current_worker().index == 0 { + debug!("Runtime SimpleAuxiliaOperator: {:?}", auxilia_operator); + } + Ok(Box::new(auxilia_operator)) } - Ok(Box::new(auxilia_operator)) } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index cb2787768d5f..41a4957ee8aa 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -13,29 +13,39 @@ //! See the License for the specific language governing permissions and //! limitations under the License. +use std::any::Any; use std::collections::BTreeMap; use std::convert::TryInto; -use std::sync::Arc; +use dyn_type::BorrowObject; use graph_proxy::apis::graph::element::GraphElement; -use graph_proxy::apis::{Direction, DynDetails, QueryParams, Statement, Vertex, ID}; +use graph_proxy::apis::{Direction, DynDetails, Edge, Element, QueryParams, Statement, Vertex, ID}; use ir_common::error::ParsePbError; use ir_common::generated::algebra as algebra_pb; use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; +use pegasus_common::downcast::*; +use pegasus_common::impl_as_any; use crate::error::{FnExecError, FnGenError, FnGenResult}; +use crate::process::entry::{DynEntry, Entry}; use crate::process::operator::map::FilterMapFuncGen; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; /// An ExpandOrIntersect operator to expand neighbors /// and intersect with the ones of the same tag found previously (if exists). /// Notice that edge_or_end_v_tag (the alias of expanded neighbors) must be specified. -struct ExpandOrIntersect> { +struct ExpandVertexOrIntersect { start_v_tag: Option, edge_or_end_v_tag: KeyId, - stmt: Box>, + stmt: Box>, +} + +struct ExpandEdgeVOrIntersect { + start_v_tag: Option, + edge_or_end_v_tag: KeyId, + stmt: Box>, } #[derive(Debug, Clone, Hash, PartialEq, PartialOrd)] @@ -44,6 +54,8 @@ pub struct Intersection { count_vec: Vec, } +impl_as_any!(Intersection); + impl Intersection { pub fn from_iter>(iter: I) -> Intersection { let mut vertex_count_map = BTreeMap::new(); @@ -103,6 +115,13 @@ impl Intersection { .zip(&self.count_vec) .flat_map(move |(vertex, count)| std::iter::repeat(vertex).take(*count as usize)) } + + pub fn drain(&mut self) -> impl Iterator + '_ { + self.vertex_vec + .drain(..) + .zip(&self.count_vec) + .flat_map(move |(vertex, count)| std::iter::repeat(vertex).take(*count as usize)) + } } impl Encode for Intersection { @@ -121,7 +140,17 @@ impl Decode for Intersection { } } -impl + 'static> FilterMapFunction for ExpandOrIntersect { +impl Element for Intersection { + fn len(&self) -> usize { + self.len() + } + + fn as_borrow_object(&self) -> BorrowObject { + BorrowObject::None + } +} + +impl FilterMapFunction for ExpandVertexOrIntersect { fn exec(&self, mut input: Record) -> FnResult> { let entry = input .get(self.start_v_tag) @@ -129,32 +158,69 @@ impl + 'static> FilterMapFunction for ExpandOrInt "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", self.start_v_tag, input )))?; - if let Some(v) = entry.as_graph_vertex() { - let id = v.id(); - let iter = self.stmt.exec(id)?.map(|e| match e.into() { - Entry::V(v) => v, - Entry::E(e) => { - Vertex::new(e.get_other_id(), e.get_other_label().cloned(), DynDetails::default()) + if let Some(id) = entry.as_id() { + let iter = self.stmt.exec(id)?; + if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { + // the case of expansion and intersection + let pre_intersection = pre_entry + .as_any_mut() + .downcast_mut::() + .ok_or(FnExecError::unexpected_data_error(&format!( + "entry is not a intersection in ExpandOrIntersect" + )))?; + pre_intersection.intersect(iter); + if pre_intersection.is_empty() { + Ok(None) + } else { + Ok(Some(input)) } - _ => { - unreachable!() + } else { + // the case of expansion only + let neighbors_intersection = Intersection::from_iter(iter); + if neighbors_intersection.is_empty() { + Ok(None) + } else { + // append columns without changing head + let columns = input.get_columns_mut(); + columns.insert(self.edge_or_end_v_tag as usize, DynEntry::new(neighbors_intersection)); + Ok(Some(input)) } - }); + } + } else { + Err(FnExecError::unsupported_error(&format!( + "expand or intersect entry {:?} of tag {:?} failed in ExpandOrIntersect", + entry, self.edge_or_end_v_tag + )))? + } + } +} + +impl FilterMapFunction for ExpandEdgeVOrIntersect { + fn exec(&self, mut input: Record) -> FnResult> { + let entry = input + .get(self.start_v_tag) + .ok_or(FnExecError::get_tag_error(&format!( + "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", + self.start_v_tag, input + )))?; + if let Some(id) = entry.as_id() { + let iter = self + .stmt + .exec(id)? + .map(|e| Vertex::new(e.get_other_id(), None, DynDetails::default())); if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { // the case of expansion and intersection - match pre_entry { - Entry::Intersection(pre_intersection) => { - pre_intersection.intersect(iter); - if pre_intersection.is_empty() { - Ok(None) - } else { - Ok(Some(input)) - } - } - _ => Err(FnExecError::unexpected_data_error(&format!( - "entry {:?} is not a intersection in ExpandOrIntersect", - pre_entry - )))?, + let pre_intersection = pre_entry + .as_any_mut() + .downcast_mut::() + .ok_or(FnExecError::unexpected_data_error(&format!( + "entry is not a intersection in ExpandOrIntersect" + )))?; + pre_intersection.intersect(iter); + if pre_intersection.is_empty() { + Ok(None) + } else { + Ok(Some(input)) } } else { // the case of expansion only @@ -164,8 +230,7 @@ impl + 'static> FilterMapFunction for ExpandOrInt } else { // append columns without changing head let columns = input.get_columns_mut(); - columns - .insert(self.edge_or_end_v_tag as usize, Arc::new(neighbors_intersection.into())); + columns.insert(self.edge_or_end_v_tag as usize, DynEntry::new(neighbors_intersection)); Ok(Some(input)) } } @@ -206,12 +271,12 @@ impl FilterMapFuncGen for algebra_pb::EdgeExpand { // Expand vertices with filters on edges. // This can be regarded as a combination of EdgeExpand (with expand_opt as Edge) + GetV let stmt = graph.prepare_explore_edge(direction, &query_params)?; - let edge_expand_operator = ExpandOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; + let edge_expand_operator = ExpandEdgeVOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; Ok(Box::new(edge_expand_operator)) } else { // Expand vertices without any filters let stmt = graph.prepare_explore_vertex(direction, &query_params)?; - let edge_expand_operator = ExpandOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; + let edge_expand_operator = ExpandVertexOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; Ok(Box::new(edge_expand_operator)) } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs index 4afcc18174df..55bb75b2719b 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs @@ -23,6 +23,7 @@ use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; +use crate::process::entry::Entry; use crate::process::operator::map::FilterMapFuncGen; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs index ea3f203ecddf..b8caeadda7d2 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs @@ -23,6 +23,7 @@ use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; +use crate::process::entry::Entry; use crate::process::operator::map::FilterMapFuncGen; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs index 66e8625a7058..b332378005a3 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs @@ -14,7 +14,6 @@ //! limitations under the License. use std::convert::{TryFrom, TryInto}; -use std::sync::Arc; use graph_proxy::utils::expr::eval::{Evaluate, Evaluator}; use ir_common::error::ParsePbError; @@ -24,9 +23,10 @@ use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use crate::error::{FnExecResult, FnGenResult}; +use crate::process::entry::DynEntry; use crate::process::operator::map::FilterMapFuncGen; use crate::process::operator::TagKey; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; /// Project entries with specified tags or further their properties. /// Notice that when projecting a single column, if the result is a None-Entry, @@ -51,11 +51,11 @@ pub enum Projector { // we may need to further distinguish the cases of none-exist tags (filtering case) and none-exist properties (output none-entry). // 2. When projecting multiple columns, even all projected columns are none-entry, the record won't be filtered for now. // This seems ambiguous. But multi-column project always appears in the end of the query. Can modify this logic if necessary. -fn exec_projector(input: &Record, projector: &Projector) -> FnExecResult> { +fn exec_projector(input: &Record, projector: &Projector) -> FnExecResult { let entry = match projector { Projector::ExprProjector(evaluator) => { - let projected_result = evaluator.eval::(Some(&input))?; - Arc::new(projected_result.into()) + let projected_result = evaluator.eval::(Some(&input))?; + DynEntry::new(projected_result) } Projector::GraphElementProjector(tag_key) => tag_key.get_arc_entry(input)?, }; @@ -161,12 +161,13 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::operator::map::FilterMapFuncGen; use crate::process::operator::tests::{ init_source, init_source_with_multi_tags, init_source_with_tag, init_vertex1, init_vertex2, to_expr_var_pb, to_expr_vars_pb, PERSON_LABEL, TAG_A, TAG_B, TAG_C, TAG_D, TAG_E, }; - use crate::process::record::{Entry, Record}; + use crate::process::record::Record; fn project_test(source: Vec, project_opr_pb: pb::Project) -> ResultStream { let conf = JobConf::new("project_test"); @@ -198,12 +199,13 @@ mod tests { let mut result = project_test(init_source(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - match res.get(None).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(None) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![object!(1), object!(2)]; assert_eq!(object_result, expected_result); @@ -225,12 +227,13 @@ mod tests { while let Some(Ok(res)) = result.next() { let a_entry = res.get(Some(TAG_A)); assert_eq!(a_entry, None); - match res.get(Some(TAG_B)).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(Some(TAG_B)) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![object!("marko"), object!("vadas")]; assert_eq!(object_result, expected_result); @@ -249,12 +252,13 @@ mod tests { let mut result = project_test(init_source_with_tag(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - match res.get(None).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(None) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![object!(1), object!(2)]; assert_eq!(object_result, expected_result); @@ -281,14 +285,17 @@ mod tests { while let Some(Ok(res)) = result.next() { // head should be None assert_eq!(res.get(None), None); - let age_val = res.get(Some(TAG_B)).unwrap(); - let name_val = res.get(Some(TAG_C)).unwrap(); - match (age_val.as_ref(), name_val.as_ref()) { - (Entry::OffGraph(age), Entry::OffGraph(name)) => { - object_result.push((age.clone(), name.clone())); - } - _ => {} - } + let age_val = res + .get(Some(TAG_B)) + .unwrap() + .as_object() + .unwrap(); + let name_val = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + object_result.push((age_val.clone(), name_val.clone())); } let expected_result = vec![(object!(29), object!("marko")), (object!(27), object!("vadas"))]; assert_eq!(object_result, expected_result); @@ -314,14 +321,17 @@ mod tests { let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - let age_val = res.get(Some(TAG_B)).unwrap(); - let name_val = res.get(Some(TAG_C)).unwrap(); - match (age_val.as_ref(), name_val.as_ref()) { - (Entry::OffGraph(age), Entry::OffGraph(name)) => { - object_result.push((age.clone(), name.clone())); - } - _ => {} - } + let age = res + .get(Some(TAG_B)) + .unwrap() + .as_object() + .unwrap(); + let name = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + object_result.push((age.clone(), name.clone())); } let expected_result = vec![(object!(29), object!("marko")), (object!(27), object!("vadas"))]; assert_eq!(object_result, expected_result); @@ -367,15 +377,22 @@ mod tests { let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - let a_age_val = res.get(Some(TAG_C)).unwrap(); - let a_name_val = res.get(Some(TAG_D)).unwrap(); - let b_name_val = res.get(Some(TAG_E)).unwrap(); - match (a_age_val.as_ref(), a_name_val.as_ref(), b_name_val.as_ref()) { - (Entry::OffGraph(a_age), Entry::OffGraph(a_name), Entry::OffGraph(b_name)) => { - object_result.push((a_age.clone(), a_name.clone(), b_name.clone())); - } - _ => {} - } + let a_age = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + let a_name = res + .get(Some(TAG_D)) + .unwrap() + .as_object() + .unwrap(); + let b_name = res + .get(Some(TAG_E)) + .unwrap() + .as_object() + .unwrap(); + object_result.push((a_age.clone(), a_name.clone(), b_name.clone())); } let expected_result = vec![ (object!(29), object!("marko"), object!("josh")), @@ -403,14 +420,13 @@ mod tests { .unwrap() .as_graph_vertex() .unwrap(); - let b_entry = res.get(Some(TAG_B)).unwrap().as_ref(); - match b_entry { - Entry::OffGraph(val) => { - a_results.push(v.id()); - b_results.push(val.clone()); - } - _ => {} - } + a_results.push(v.id()); + let b_val = res + .get(Some(TAG_B)) + .unwrap() + .as_object() + .unwrap(); + b_results.push(b_val.clone()); } let expected_a_result = vec![1, 2]; let expected_b_result = vec![object!(29), object!(27)]; @@ -437,14 +453,17 @@ mod tests { let mut result = project_test(init_source(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - let age_val = res.get(Some(TAG_B)).unwrap(); - let name_val = res.get(Some(TAG_C)).unwrap(); - match (age_val.as_ref(), name_val.as_ref()) { - (Entry::OffGraph(age), Entry::OffGraph(name)) => { - object_result.push((age.clone(), name.clone())); - } - _ => {} - } + let age = res + .get(Some(TAG_B)) + .unwrap() + .as_object() + .unwrap(); + let name = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + object_result.push((age.clone(), name.clone())); } let expected_result = vec![(object!(29), object!("marko")), (object!(27), object!("vadas"))]; assert_eq!(object_result, expected_result); @@ -492,12 +511,13 @@ mod tests { let mut result = project_test(init_source(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - match res.get(None).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(None) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![ object!(vec![object!(29), object!("marko")]), @@ -519,12 +539,13 @@ mod tests { let mut result = project_test(init_source(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - match res.get(None).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(None) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![ Object::KV( @@ -566,12 +587,13 @@ mod tests { let mut result = project_test(init_source_with_tag(), project_opr_pb); let mut object_result = vec![]; while let Some(Ok(res)) = result.next() { - match res.get(None).unwrap().as_ref() { - Entry::OffGraph(val) => { - object_result.push(val.clone()); - } - _ => {} - } + object_result.push( + res.get(None) + .unwrap() + .as_object() + .unwrap() + .clone(), + ); } let expected_result = vec![ @@ -613,10 +635,16 @@ mod tests { let mut result = project_test(init_source_with_multi_tags(), project_opr_pb); let mut results = vec![]; while let Some(Ok(res)) = result.next() { - let a_entry = res.get(Some(TAG_C)).unwrap().as_ref(); - let b_entry = res.get(Some(TAG_D)).unwrap().as_ref(); - let v1 = a_entry.as_graph_vertex().unwrap(); - let v2 = b_entry.as_graph_vertex().unwrap(); + let v1 = res + .get(Some(TAG_C)) + .unwrap() + .as_graph_vertex() + .unwrap(); + let v2 = res + .get(Some(TAG_D)) + .unwrap() + .as_graph_vertex() + .unwrap(); results.push(v1.id()); results.push(v2.id()); } @@ -682,10 +710,18 @@ mod tests { result_cnt += 1; // head should be None assert_eq!(res.get(None), None); - let c_val = res.get(Some(TAG_C)).unwrap(); - let d_val = res.get(Some(TAG_D)).unwrap(); - match (c_val.as_ref(), d_val.as_ref()) { - (Entry::OffGraph(Object::None), Entry::OffGraph(Object::None)) => { + let c_val = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + let d_val = res + .get(Some(TAG_D)) + .unwrap() + .as_object() + .unwrap(); + match (c_val, d_val) { + (Object::None, Object::None) => { assert!(true) } _ => { @@ -718,10 +754,18 @@ mod tests { result_cnt += 1; // head should be None assert_eq!(res.get(None), None); - let c_val = res.get(Some(TAG_C)).unwrap(); - let d_val = res.get(Some(TAG_D)).unwrap(); - match (c_val.as_ref(), d_val.as_ref()) { - (Entry::OffGraph(Object::None), Entry::OffGraph(Object::None)) => { + let c_val = res + .get(Some(TAG_C)) + .unwrap() + .as_object() + .unwrap(); + let d_val = res + .get(Some(TAG_D)) + .unwrap() + .as_object() + .unwrap(); + match (c_val, d_val) { + (Object::None, Object::None) => { assert!(true) } _ => { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs b/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs index 715b9dcc8a40..3faf217c80dc 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs @@ -27,7 +27,6 @@ pub mod source; pub mod subtask; use std::convert::TryFrom; -use std::sync::Arc; use dyn_type::Object; use graph_proxy::apis::{Details, Element, PropKey}; @@ -37,7 +36,8 @@ use ir_common::{KeyId, NameOrId}; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use crate::error::{FnExecError, FnExecResult}; -use crate::process::record::{Entry, Record}; +use crate::process::entry::DynEntry; +use crate::process::record::Record; #[derive(Clone, Debug, Default)] pub struct TagKey { @@ -47,89 +47,83 @@ pub struct TagKey { impl TagKey { /// This is for key generation, which generate the key of the input Record according to the tag_key field - pub fn get_arc_entry(&self, input: &Record) -> FnExecResult> { + pub fn get_arc_entry(&self, input: &Record) -> FnExecResult { if let Some(entry) = input.get(self.tag) { if let Some(prop_key) = self.key.as_ref() { let prop = self.get_key(entry, prop_key)?; - Ok(Arc::new(prop)) + Ok(prop) } else { Ok(entry.clone()) } } else { - Ok(Arc::new((Object::None).into())) + Ok(DynEntry::new(Object::None)) } } - /// This is for accum, which get the entry of the input Record according to the tag_key field - pub fn get_entry(&self, input: &Record) -> FnExecResult { - if let Some(entry) = input.get(self.tag) { - if let Some(prop_key) = self.key.as_ref() { - Ok(self.get_key(entry, prop_key)?) - } else { - Ok(entry.as_ref().clone()) - } + fn get_key(&self, entry: &DynEntry, prop_key: &PropKey) -> FnExecResult { + if let PropKey::Len = prop_key { + let obj: Object = (entry.len() as u64).into(); + Ok(DynEntry::new(obj)) } else { - Ok((Object::None).into()) - } - } + if let Some(element) = entry.as_graph_element() { + let prop_obj = match prop_key { + PropKey::Id => element.id().into(), + PropKey::Label => element + .label() + .map(|label| label.into()) + .unwrap_or(Object::None), + PropKey::Len => unreachable!(), + PropKey::All => { + let details = element + .details() + .ok_or(FnExecError::unexpected_data_error(&format!( + "Get `PropKey::All` on {:?}", + entry, + )))?; - fn get_key(&self, entry: &Arc, prop_key: &PropKey) -> FnExecResult { - if let Some(element) = entry.as_graph_element() { - let prop_obj = match prop_key { - PropKey::Id => element.id().into(), - PropKey::Label => element - .label() - .map(|label| label.into()) - .unwrap_or(Object::None), - PropKey::Len => (element.len() as u64).into(), - PropKey::All => { - let details = element - .details() - .ok_or(FnExecError::unexpected_data_error(&format!( - "Get `PropKey::All` on {:?}", - entry, - )))?; - - if let Some(properties) = details.get_all_properties() { - properties - .into_iter() - .map(|(key, value)| { - let obj_key: Object = match key { - NameOrId::Str(str) => str.into(), - NameOrId::Id(id) => id.into(), - }; - (obj_key, value) - }) - .collect::>() - .into() - } else { - Object::None + if let Some(properties) = details.get_all_properties() { + properties + .into_iter() + .map(|(key, value)| { + let obj_key: Object = match key { + NameOrId::Str(str) => str.into(), + NameOrId::Id(id) => id.into(), + }; + (obj_key, value) + }) + .collect::>() + .into() + } else { + Object::None + } } - } - PropKey::Key(key) => { - let details = element - .details() - .ok_or(FnExecError::unexpected_data_error(&format!( - "Get `PropKey::Key` of {:?} on {:?}", - key, entry, - )))?; - if let Some(properties) = details.get_property(key) { - properties - .try_to_owned() - .ok_or(FnExecError::unexpected_data_error("unable to own the `BorrowObject`"))? - } else { - Object::None + PropKey::Key(key) => { + let details = element + .details() + .ok_or(FnExecError::unexpected_data_error(&format!( + "Get `PropKey::Key` of {:?} on {:?}", + key, entry, + )))?; + if let Some(properties) = details.get_property(key) { + properties + .try_to_owned() + .ok_or(FnExecError::unexpected_data_error( + "unable to own the `BorrowObject`", + ))? + } else { + Object::None + } } - } - }; + }; - Ok(prop_obj.into()) - } else { - Err(FnExecError::unexpected_data_error(&format!( - " + Ok(DynEntry::new(prop_obj)) + } else { + Err(FnExecError::unexpected_data_error(&format!( + " Get key failed since get details from a none-graph element {:?} ", - entry - ))) + entry + ))) + } } } } @@ -199,6 +193,7 @@ pub(crate) mod tests { use ir_common::{KeyId, LabelId}; use super::*; + use crate::process::entry::Entry; pub const TAG_A: KeyId = 0; pub const TAG_B: KeyId = 1; @@ -302,9 +297,9 @@ pub(crate) mod tests { fn test_get_none_tag_entry() { let tag_key = TagKey { tag: None, key: None }; let record = init_record(); - let expected = object!(10).into(); + let expected = object!(10); let entry = tag_key.get_arc_entry(&record).unwrap(); - assert_eq!(entry.as_ref().clone(), expected) + assert_eq!(entry.as_object().unwrap().clone(), expected) } #[test] @@ -325,14 +320,13 @@ pub(crate) mod tests { let tag_key = TagKey { tag: Some((0 as KeyId).into()), key: Some(PropKey::Key("age".into())) }; let expected = 27; let record = init_record(); - let entry = tag_key.get_arc_entry(&record).unwrap(); - match entry.as_ref() { - Entry::OffGraph(obj) => { - assert_eq!(obj.clone(), object!(expected)); - } - _ => { - assert!(false); - } - } + let entry = tag_key + .get_arc_entry(&record) + .unwrap() + .as_object() + .unwrap() + .clone(); + + assert_eq!(entry, object!(expected)); } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs index b73d5910da05..85361aad257c 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs @@ -16,13 +16,14 @@ use std::convert::TryInto; use std::sync::Arc; -use graph_proxy::apis::{GraphElement, Partitioner, VertexOrEdge}; +use graph_proxy::apis::{GraphElement, Partitioner}; use ir_common::error::ParsePbError; use ir_common::generated::common as common_pb; use ir_common::KeyId; use pegasus::api::function::{FnResult, RouteFunction}; use crate::error::FnExecError; +use crate::process::entry::{Entry, EntryDataType}; use crate::process::record::Record; pub struct RecordRouter { @@ -49,30 +50,35 @@ impl RecordRouter { impl RouteFunction for RecordRouter { fn route(&self, t: &Record) -> FnResult { if let Some(entry) = t.get(self.shuffle_key.clone()) { - if let Some(v) = entry.as_graph_vertex() { - Ok(self - .p - .get_partition(&v.id(), self.num_workers)?) - } else if let Some(e) = entry.as_graph_edge() { - // shuffle e to the partition that contains other_id - Ok(self - .p - .get_partition(&e.get_other_id(), self.num_workers)?) - } else if let Some(p) = entry.as_graph_path() { - let path_end = p - .get_path_end() - .ok_or(FnExecError::unexpected_data_error("get path_end failed in shuffle"))?; - match path_end { - VertexOrEdge::V(v) => Ok(self + // TODO: this may occur bugs if `Id` is actually an edge entry. + match entry.get_type() { + EntryDataType::Id | EntryDataType::V => { + let id = entry + .as_id() + .ok_or(FnExecError::unexpected_data_error("get id failed in shuffle"))?; + Ok(self.p.get_partition(&id, self.num_workers)?) + } + EntryDataType::E => { + let e = entry + .as_graph_edge() + .ok_or(FnExecError::unexpected_data_error("get edge failed in shuffle"))?; + Ok(self .p - .get_partition(&v.id(), self.num_workers)?), - VertexOrEdge::E(e) => Ok(self + .get_partition(&e.src_id, self.num_workers)?) + } + EntryDataType::P => { + let p = entry + .as_graph_path() + .ok_or(FnExecError::unexpected_data_error("get path failed in shuffle"))?; + let path_end = p + .get_path_end() + .ok_or(FnExecError::unexpected_data_error("get path_end failed in shuffle"))?; + Ok(self .p - .get_partition(&e.get_other_id(), self.num_workers)?), + .get_partition(&path_end.id(), self.num_workers)?) } - } else { - //TODO(bingqing): deal with other element shuffle - Ok(0) + // TODO: + _ => Ok(0), } } else { Ok(0) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs index de1b8352fb3b..ca3add9d8a22 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs @@ -16,21 +16,23 @@ use std::borrow::BorrowMut; use std::collections::HashMap; use std::convert::TryInto; -use std::ops::Deref; use dyn_type::Object; -use graph_proxy::apis::{Edge, GraphElement, GraphPath, Vertex, VertexOrEdge}; +use graph_proxy::apis::{Edge, Element, GraphElement, GraphPath, Vertex}; use ir_common::generated::algebra as algebra_pb; use ir_common::generated::algebra::sink_default::MetaType; use ir_common::generated::common as common_pb; use ir_common::generated::results as result_pb; use ir_common::{KeyId, NameOrId}; use pegasus::api::function::{FnResult, MapFunction}; +use pegasus_common::downcast::AsAny; use prost::Message; use crate::error::FnGenResult; +use crate::process::entry::{CollectionEntry, DynEntry, Entry, EntryDataType}; +use crate::process::operator::map::Intersection; use crate::process::operator::sink::{SinkGen, Sinker}; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; #[derive(Debug)] pub struct RecordSinkEncoder { @@ -41,11 +43,15 @@ pub struct RecordSinkEncoder { } impl RecordSinkEncoder { - fn entry_to_pb(&self, e: &Entry) -> result_pb::Entry { - let inner = match e { - Entry::Collection(collection) => { + fn entry_to_pb(&self, e: &DynEntry) -> result_pb::Entry { + let inner = match e.get_type() { + EntryDataType::Collection => { + let collection = e + .as_any_ref() + .downcast_ref::() + .unwrap(); let mut collection_pb = Vec::with_capacity(collection.len()); - for element in collection.into_iter() { + for element in &collection.inner { let element_pb = self.element_to_pb(element); collection_pb.push(element_pb); } @@ -53,7 +59,11 @@ impl RecordSinkEncoder { collection: collection_pb, })) } - Entry::Intersection(intersection) => { + EntryDataType::Intersect => { + let intersection = e + .as_any_ref() + .downcast_ref::() + .unwrap(); let mut collection_pb = Vec::with_capacity(intersection.len()); for v in intersection.iter() { let vertex_pb = self.vertex_to_pb(v); @@ -73,30 +83,33 @@ impl RecordSinkEncoder { result_pb::Entry { inner } } - fn element_to_pb(&self, e: &Entry) -> result_pb::Element { - let inner = match e { - Entry::V(v) => { - let vertex_pb = self.vertex_to_pb(v); + fn element_to_pb(&self, e: &DynEntry) -> result_pb::Element { + let inner = match e.get_type() { + EntryDataType::V => { + let vertex_pb = self.vertex_to_pb(e.as_graph_vertex().unwrap()); Some(result_pb::element::Inner::Vertex(vertex_pb)) } - Entry::E(e) => { - let edge_pb = self.edge_to_pb(e); + EntryDataType::E => { + let edge_pb = self.edge_to_pb(e.as_graph_edge().unwrap()); Some(result_pb::element::Inner::Edge(edge_pb)) } - Entry::P(p) => { - let path_pb = self.path_to_pb(p); + EntryDataType::P => { + let path_pb = self.path_to_pb(e.as_graph_path().unwrap()); Some(result_pb::element::Inner::GraphPath(path_pb)) } - Entry::OffGraph(o) => { - let obj_pb = self.object_to_pb(o.clone()); + EntryDataType::Obj => { + let obj_pb = self.object_to_pb(e.as_object().unwrap().clone()); Some(result_pb::element::Inner::Object(obj_pb)) } - Entry::Collection(_) => { + EntryDataType::Collection => { unreachable!() } - Entry::Intersection(_) => { + EntryDataType::Intersect => { unreachable!() } + _ => { + todo!() + } }; result_pb::Element { inner } } @@ -170,20 +183,10 @@ impl RecordSinkEncoder { } } - fn vertex_or_edge_to_pb(&self, vertex_or_edge: &VertexOrEdge) -> result_pb::graph_path::VertexOrEdge { - match vertex_or_edge { - VertexOrEdge::V(v) => { - let vertex_pb = self.vertex_to_pb(v); - result_pb::graph_path::VertexOrEdge { - inner: Some(result_pb::graph_path::vertex_or_edge::Inner::Vertex(vertex_pb)), - } - } - VertexOrEdge::E(e) => { - let edge_pb = self.edge_to_pb(e); - result_pb::graph_path::VertexOrEdge { - inner: Some(result_pb::graph_path::vertex_or_edge::Inner::Edge(edge_pb)), - } - } + fn vertex_or_edge_to_pb(&self, vertex_or_edge: &Vertex) -> result_pb::graph_path::VertexOrEdge { + let vertex_pb = self.vertex_to_pb(vertex_or_edge); + result_pb::graph_path::VertexOrEdge { + inner: Some(result_pb::graph_path::vertex_or_edge::Inner::Vertex(vertex_pb)), } } @@ -210,7 +213,7 @@ impl MapFunction> for RecordSinkEncoder { let mut sink_columns = Vec::with_capacity(self.sink_keys.len()); for sink_key in self.sink_keys.iter() { if let Some(entry) = input.get(sink_key.clone()) { - let entry_pb = self.entry_to_pb(entry.deref()); + let entry_pb = self.entry_to_pb(entry); let column_pb = result_pb::Column { name_or_id: sink_key .clone() diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs index 7357a81de053..7fb16b66a698 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs @@ -25,6 +25,7 @@ use ir_common::generated::schema as schema_pb; use ir_common::KeyId; use crate::error::{FnExecError, FnExecResult, FnGenResult}; +use crate::process::entry::Entry; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::sink::{SinkGen, Sinker}; use crate::process::record::Record; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs index cfd092e0fd4f..3b98e5822a65 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs @@ -93,6 +93,7 @@ mod tests { use pegasus::result::ResultStream; use pegasus::JobConf; + use crate::process::entry::Entry; use crate::process::operator::sort::CompareFunctionGen; use crate::process::operator::tests::{ init_source, init_source_with_tag, to_var_pb, PERSON_LABEL, TAG_A, diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/subtask/apply.rs b/interactive_engine/executor/ir/runtime/src/process/operator/subtask/apply.rs index 43111764ecd1..8fec26ec0f35 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/subtask/apply.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/subtask/apply.rs @@ -14,7 +14,6 @@ //! limitations under the License. use std::convert::TryFrom; -use std::sync::Arc; use dyn_type::Object; use ir_common::generated::algebra as algebra_pb; @@ -23,8 +22,9 @@ use ir_common::KeyId; use pegasus::api::function::{BinaryFunction, FnResult}; use crate::error::{FnExecError, FnGenError, FnGenResult}; +use crate::process::entry::DynEntry; use crate::process::functions::ApplyGen; -use crate::process::record::{Entry, Record}; +use crate::process::record::Record; #[derive(Debug)] struct ApplyOperator { @@ -59,7 +59,7 @@ impl BinaryFunction, Option> for ApplyOperator { } JoinKind::LeftOuter => { if sub.is_empty() { - let entry: Arc = Arc::new(Object::None.into()); + let entry = DynEntry::new(Object::None); if let Some(alias) = self.alias.as_ref() { let columns = parent.get_columns_mut(); columns.insert(*alias as usize, entry.clone()); diff --git a/interactive_engine/executor/ir/runtime/src/process/record.rs b/interactive_engine/executor/ir/runtime/src/process/record.rs index d9f64cc34589..76114da7e7c4 100644 --- a/interactive_engine/executor/ir/runtime/src/process/record.rs +++ b/interactive_engine/executor/ir/runtime/src/process/record.rs @@ -14,87 +14,30 @@ //! limitations under the License. use std::borrow::BorrowMut; -use std::convert::{TryFrom, TryInto}; use std::hash::Hash; -use std::ops::Add; -use std::sync::Arc; -use dyn_type::{BorrowObject, Object}; -use graph_proxy::apis::{Edge, Element, GraphElement, GraphPath, Vertex, VertexOrEdge}; use graph_proxy::utils::expr::eval::Context; -use ir_common::error::ParsePbError; -use ir_common::generated::results as result_pb; use ir_common::{KeyId, NameOrId}; use pegasus::api::function::DynIter; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use vec_map::VecMap; -use crate::process::operator::map::Intersection; - -#[derive(Debug, Clone, Hash, PartialEq, PartialOrd)] -pub enum Entry { - V(Vertex), - E(Edge), - P(GraphPath), - OffGraph(Object), - Intersection(Intersection), - Collection(Vec), -} - -impl Entry { - pub fn as_graph_vertex(&self) -> Option<&Vertex> { - match self { - Entry::V(v) => Some(v), - _ => None, - } - } - - pub fn as_graph_edge(&self) -> Option<&Edge> { - match self { - Entry::E(e) => Some(e), - _ => None, - } - } - - pub fn as_graph_path(&self) -> Option<&GraphPath> { - match self { - Entry::P(p) => Some(p), - _ => None, - } - } - - pub fn as_object(&self) -> Option<&Object> { - match self { - Entry::OffGraph(object) => Some(object), - _ => None, - } - } - - pub fn as_graph_path_mut(&mut self) -> Option<&mut GraphPath> { - match self { - Entry::P(graph_path) => Some(graph_path), - _ => None, - } - } - - pub fn is_none(&self) -> bool { - match self { - Entry::OffGraph(Object::None) => true, - _ => false, - } - } -} +use crate::process::entry::{DynEntry, Entry, EntryDataType}; +use graph_proxy::apis::{GraphPath, Vertex}; #[derive(Debug, Clone, Default)] pub struct Record { - curr: Option>, - columns: VecMap>, + curr: Option, + columns: VecMap, } +unsafe impl Send for Record {} +unsafe impl Sync for Record {} + impl Record { - pub fn new>(entry: E, tag: Option) -> Self { - let entry = Arc::new(entry.into()); + pub fn new(entry: E, tag: Option) -> Self { let mut columns = VecMap::new(); + let entry = DynEntry::new(entry); if let Some(tag) = tag { columns.insert(tag as usize, entry.clone()); } @@ -102,11 +45,19 @@ impl Record { } /// A handy api to append entry of different types that can be turned into `Entry` - pub fn append>(&mut self, entry: E, alias: Option) { - self.append_arc_entry(Arc::new(entry.into()), alias) + pub fn append(&mut self, entry: E, alias: Option) { + let entry = DynEntry::new(entry); + self.append_arc_entry(entry, alias) + } + + // this is a test for alias necessary opt: move current into columns with head_alias, and append new entry as current + pub fn append_with_head_alias(&mut self, entry: E, head_alias: Option) { + self.columns + .insert(head_alias.unwrap() as usize, self.curr.clone().unwrap()); + self.curr = Some(DynEntry::new(entry)); } - pub fn append_arc_entry(&mut self, entry: Arc, alias: Option) { + pub fn append_arc_entry(&mut self, entry: DynEntry, alias: Option) { if let Some(alias) = alias { self.columns .insert(alias as usize, entry.clone()); @@ -115,22 +66,22 @@ impl Record { } /// Set new current entry for the record - pub fn set_curr_entry(&mut self, entry: Option>) { + pub fn set_curr_entry(&mut self, entry: Option) { self.curr = entry; } - pub fn get_column_mut(&mut self, tag: &KeyId) -> Option<&mut Entry> { + pub fn get_column_mut(&mut self, tag: &KeyId) -> Option<&mut dyn Entry> { self.columns .get_mut(*tag as usize) - .map(|e| Arc::get_mut(e)) + .map(|e| e.get_mut()) .unwrap_or(None) } - pub fn get_columns_mut(&mut self) -> &mut VecMap> { + pub fn get_columns_mut(&mut self) -> &mut VecMap { self.columns.borrow_mut() } - pub fn get(&self, tag: Option) -> Option<&Arc> { + pub fn get(&self, tag: Option) -> Option<&DynEntry> { if let Some(tag) = tag { self.columns.get(tag as usize) } else { @@ -138,7 +89,7 @@ impl Record { } } - pub fn take(&mut self, tag: Option<&KeyId>) -> Option> { + pub fn take(&mut self, tag: Option<&KeyId>) -> Option { if let Some(tag) = tag { self.columns.remove(*tag as usize) } else { @@ -171,47 +122,8 @@ impl Record { } } -impl Into for Vertex { - fn into(self) -> Entry { - Entry::V(self) - } -} - -impl Into for Edge { - fn into(self) -> Entry { - Entry::E(self) - } -} - -impl Into for GraphPath { - fn into(self) -> Entry { - Entry::P(self) - } -} - -impl Into for VertexOrEdge { - fn into(self) -> Entry { - match self { - VertexOrEdge::V(v) => v.into(), - VertexOrEdge::E(e) => e.into(), - } - } -} - -impl Into for Object { - fn into(self) -> Entry { - Entry::OffGraph(self) - } -} - -impl Into for Intersection { - fn into(self) -> Entry { - Entry::Intersection(self) - } -} - -impl Context for Record { - fn get(&self, tag: Option<&NameOrId>) -> Option<&Entry> { +impl Context for Record { + fn get(&self, tag: Option<&NameOrId>) -> Option<&DynEntry> { let tag = if let Some(tag) = tag { match tag { // TODO: may better throw an unsupported error if tag is a string_tag @@ -222,68 +134,33 @@ impl Context for Record { None }; self.get(tag) - .map(|entry| match entry.as_ref() { - Entry::Collection(_) => None, - Entry::Intersection(_) => None, - _ => Some(entry.as_ref()), + .map(|entry| { + let entry_type = entry.get_type(); + match entry_type { + EntryDataType::Collection => None, + EntryDataType::Intersect => None, + _ => Some(entry), + } }) .unwrap_or(None) } } -impl Element for Entry { - fn as_graph_element(&self) -> Option<&dyn GraphElement> { - match self { - Entry::V(v) => v.as_graph_element(), - Entry::E(e) => e.as_graph_element(), - Entry::P(p) => p.as_graph_element(), - Entry::OffGraph(_) => None, - Entry::Collection(_) => None, - Entry::Intersection(_) => None, - } - } - - fn len(&self) -> usize { - match self { - Entry::V(v) => v.len(), - Entry::E(e) => e.len(), - Entry::P(p) => p.len(), - Entry::OffGraph(obj) => obj.len(), - Entry::Collection(c) => c.len(), - Entry::Intersection(i) => i.len(), - } - } - - fn as_borrow_object(&self) -> BorrowObject { - match self { - Entry::V(v) => v.as_borrow_object(), - Entry::E(e) => e.as_borrow_object(), - Entry::P(p) => p.as_borrow_object(), - Entry::OffGraph(obj) => obj.as_borrow(), - Entry::Collection(_) => unreachable!(), - Entry::Intersection(_) => unreachable!(), - } - } -} - /// RecordKey is the key fields of a Record, with each key corresponding to a request column_tag -#[derive(Clone, Debug, Hash, PartialEq)] +#[derive(Clone, Debug, Hash, PartialEq, Eq)] pub struct RecordKey { - key_fields: Vec>, + key_fields: Vec, } impl RecordKey { - pub fn new(key_fields: Vec>) -> Self { + pub fn new(key_fields: Vec) -> Self { RecordKey { key_fields } } - pub fn take(self) -> Vec> { + pub fn take(self) -> Vec { self.key_fields } } -impl Eq for RecordKey {} -impl Eq for Entry {} - pub struct RecordExpandIter { tag: Option, origin: Record, @@ -296,7 +173,7 @@ impl RecordExpandIter { } } -impl> Iterator for RecordExpandIter { +impl Iterator for RecordExpandIter { type Item = Record; fn next(&mut self) -> Option { @@ -323,7 +200,7 @@ impl RecordPathExpandIter { } } -impl> Iterator for RecordPathExpandIter { +impl Iterator for RecordPathExpandIter { type Item = Record; fn next(&mut self) -> Option { @@ -331,22 +208,14 @@ impl> Iterator for RecordPathExpandIter { let mut curr_path = self.curr_path.clone(); loop { match self.children.next() { - Some(elem) => { - let graph_obj = elem.into(); - match graph_obj { - Entry::V(v) => { - if curr_path.append(v) { - record.append(curr_path, None); - return Some(record); - } - } - Entry::E(e) => { - if curr_path.append(e) { - record.append(curr_path, None); - return Some(record); - } + Some(mut elem) => { + // currently, we only support GraphPath containing vertices. + if let Some(vertex) = elem.as_any_mut().downcast_mut::() { + let v = std::mem::replace(vertex, Default::default()); + if curr_path.append(v) { + record.append(curr_path, None); + return Some(record); } - _ => {} } } None => return None, @@ -355,71 +224,6 @@ impl> Iterator for RecordPathExpandIter { } } -impl Encode for Entry { - fn write_to(&self, writer: &mut W) -> std::io::Result<()> { - match self { - Entry::V(v) => { - writer.write_u8(0)?; - v.write_to(writer)?; - } - Entry::E(e) => { - writer.write_u8(1)?; - e.write_to(writer)?; - } - Entry::P(p) => { - writer.write_u8(2)?; - p.write_to(writer)?; - } - Entry::OffGraph(obj) => { - writer.write_u8(3)?; - obj.write_to(writer)?; - } - Entry::Collection(collection) => { - writer.write_u8(4)?; - collection.write_to(writer)? - } - Entry::Intersection(intersection) => { - writer.write_u8(5)?; - intersection.write_to(writer)? - } - } - Ok(()) - } -} - -impl Decode for Entry { - fn read_from(reader: &mut R) -> std::io::Result { - let opt = reader.read_u8()?; - match opt { - 0 => { - let v = ::read_from(reader)?; - Ok(Entry::V(v)) - } - 1 => { - let e = ::read_from(reader)?; - Ok(Entry::E(e)) - } - 2 => { - let p = ::read_from(reader)?; - Ok(Entry::P(p)) - } - 3 => { - let obj = ::read_from(reader)?; - Ok(Entry::OffGraph(obj)) - } - 4 => { - let collection = >::read_from(reader)?; - Ok(Entry::Collection(collection)) - } - 5 => { - let intersection = ::read_from(reader)?; - Ok(Entry::Intersection(intersection)) - } - _ => Err(std::io::Error::new(std::io::ErrorKind::Other, "unreachable")), - } - } -} - impl Encode for Record { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { match &self.curr { @@ -443,13 +247,13 @@ impl Encode for Record { impl Decode for Record { fn read_from(reader: &mut R) -> std::io::Result { let opt = reader.read_u8()?; - let curr = if opt == 0 { None } else { Some(Arc::new(::read_from(reader)?)) }; + let curr = if opt == 0 { None } else { Some(::read_from(reader)?) }; let size = ::read_from(reader)? as usize; let mut columns = VecMap::with_capacity(size); for _i in 0..size { let k = ::read_from(reader)? as usize; - let v = ::read_from(reader)?; - columns.insert(k, Arc::new(v)); + let v = ::read_from(reader)?; + columns.insert(k, v); } Ok(Record { curr, columns }) } @@ -459,7 +263,7 @@ impl Encode for RecordKey { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { writer.write_u32(self.key_fields.len() as u32)?; for key in self.key_fields.iter() { - (&**key).write_to(writer)? + key.write_to(writer)? } Ok(()) } @@ -470,60 +274,9 @@ impl Decode for RecordKey { let len = reader.read_u32()?; let mut key_fields = Vec::with_capacity(len as usize); for _i in 0..len { - let entry = ::read_from(reader)?; - key_fields.push(Arc::new(entry)) + let entry = ::read_from(reader)?; + key_fields.push(entry) } Ok(RecordKey { key_fields }) } } - -impl TryFrom for Entry { - type Error = ParsePbError; - - fn try_from(entry_pb: result_pb::Entry) -> Result { - if let Some(inner) = entry_pb.inner { - match inner { - result_pb::entry::Inner::Element(e) => Ok(e.try_into()?), - result_pb::entry::Inner::Collection(c) => Ok(Entry::Collection( - c.collection - .into_iter() - .map(|e| e.try_into()) - .collect::, Self::Error>>()?, - )), - } - } else { - Err(ParsePbError::EmptyFieldError("entry inner is empty".to_string()))? - } - } -} - -impl TryFrom for Entry { - type Error = ParsePbError; - fn try_from(e: result_pb::Element) -> Result { - if let Some(inner) = e.inner { - match inner { - result_pb::element::Inner::Vertex(v) => Ok(Entry::V(v.try_into()?)), - result_pb::element::Inner::Edge(e) => Ok(Entry::E(e.try_into()?)), - result_pb::element::Inner::GraphPath(p) => Ok(Entry::P(p.try_into()?)), - result_pb::element::Inner::Object(o) => Ok(Entry::OffGraph(o.try_into()?)), - } - } else { - Err(ParsePbError::EmptyFieldError("element inner is empty".to_string()))? - } - } -} - -impl Add for Entry { - type Output = Entry; - - fn add(self, rhs: Self) -> Self::Output { - match (self, rhs) { - (Entry::OffGraph(o1), Entry::OffGraph(o2)) => match (o1, o2) { - (o, Object::None) | (Object::None, o) => o.into(), - (Object::Primitive(p1), Object::Primitive(p2)) => Object::Primitive(p1.add(p2)).into(), - _ => Object::None.into(), - }, - _ => Object::None.into(), - } - } -} From f9309b263c394a4ae3d2b773e8f2191a6766a5fa Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 6 Dec 2022 12:15:42 +0800 Subject: [PATCH 2/8] [GIE/Runtime] fix ci tests. --- interactive_engine/executor/ir/integrated/tests/expand_test.rs | 2 -- .../ir/runtime/src/process/operator/sink/sink_vineyard.rs | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index 3f3264f217e7..36aa34e9cdf7 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -759,7 +759,6 @@ mod test { let intersection = record .get(Some(TAG_C)) .unwrap() - // .inner .as_any_ref() .downcast_ref::() .unwrap(); @@ -837,7 +836,6 @@ mod test { let intersection = record .get(Some(TAG_C)) .unwrap() - // .inner .as_any_ref() .downcast_ref::() .unwrap(); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs index 7fb16b66a698..033743ee0c45 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs @@ -174,6 +174,7 @@ mod tests { use pegasus::JobConf; use crate::error::FnExecResult; + use crate::process::entry::Entry; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::tests::{init_source, init_vertex1, init_vertex2, PERSON_LABEL}; use crate::process::record::Record; From 333d6cd0371750791d55f62fcf75846a7f74c667 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 6 Dec 2022 16:30:24 +0800 Subject: [PATCH 3/8] [GIE/Runtime] make EntryType of `Id` as `Vid` explicitly --- .../ir/integrated/tests/expand_test.rs | 2 +- .../executor/ir/runtime/src/process/entry.rs | 60 ++++-------- .../src/process/operator/accum/accum.rs | 2 +- .../process/operator/flatmap/edge_expand.rs | 92 +++++++++---------- .../src/process/operator/map/auxilia.rs | 9 +- .../process/operator/map/expand_intersect.rs | 4 +- .../runtime/src/process/operator/shuffle.rs | 5 +- .../executor/ir/runtime/src/process/record.rs | 2 +- 8 files changed, 76 insertions(+), 100 deletions(-) diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index 36aa34e9cdf7..39e9beed9cea 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -32,6 +32,7 @@ mod test { use pegasus::api::{Map, Sink}; use pegasus::result::ResultStream; use pegasus::JobConf; + use pegasus_common::downcast::AsAny; use runtime::process::entry::Entry; use runtime::process::operator::flatmap::FlatMapFuncGen; use runtime::process::operator::map::{FilterMapFuncGen, Intersection}; @@ -39,7 +40,6 @@ mod test { use runtime::process::record::Record; use crate::common::test::*; - use pegasus_common::downcast::AsAny; // g.V() fn source_gen(alias: Option) -> Box + Send> { diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs index 449098e1770a..63d1c11babe6 100644 --- a/interactive_engine/executor/ir/runtime/src/process/entry.rs +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -35,8 +35,7 @@ use crate::process::operator::map::Intersection; #[derive(Debug)] pub enum EntryDataType { - // TODO: A global id to denote current data; Currently, it is mostly vertex global id?? - Id, + Vid, // A vertex global id V, E, P, @@ -48,9 +47,9 @@ pub enum EntryDataType { impl PartialEq for EntryDataType { fn eq(&self, other: &Self) -> bool { match (self, other) { - (EntryDataType::Id, EntryDataType::Id) - | (EntryDataType::Id, EntryDataType::V) - | (EntryDataType::V, EntryDataType::Id) + (EntryDataType::Vid, EntryDataType::Vid) + | (EntryDataType::Vid, EntryDataType::V) + | (EntryDataType::V, EntryDataType::Vid) | (EntryDataType::V, EntryDataType::V) | (EntryDataType::E, EntryDataType::E) | (EntryDataType::P, EntryDataType::P) @@ -64,7 +63,9 @@ impl PartialEq for EntryDataType { pub trait Entry: Debug + Send + Sync + AsAny + Element { fn get_type(&self) -> EntryDataType; - fn as_id(&self) -> Option; + fn as_vid(&self) -> Option { + None + } fn as_graph_vertex(&self) -> Option<&Vertex> { None } @@ -123,8 +124,8 @@ impl Entry for DynEntry { self.inner.get_type() } - fn as_id(&self) -> Option { - self.inner.as_id() + fn as_vid(&self) -> Option { + self.inner.as_vid() } fn as_graph_vertex(&self) -> Option<&Vertex> { @@ -148,9 +149,9 @@ impl Encode for DynEntry { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { let entry_type = self.get_type(); match entry_type { - EntryDataType::Id => { + EntryDataType::Vid => { writer.write_u8(0)?; - write_id(writer, self.as_id().unwrap())?; + write_id(writer, self.as_vid().unwrap())?; } EntryDataType::V => { writer.write_u8(1)?; @@ -245,8 +246,7 @@ impl Element for DynEntry { impl GraphElement for DynEntry { fn id(&self) -> ID { match self.get_type() { - EntryDataType::Id => self.as_id().unwrap(), - EntryDataType::V => self.as_graph_vertex().unwrap().id(), + EntryDataType::Vid | EntryDataType::V => self.as_vid().unwrap(), EntryDataType::E => self.as_graph_edge().unwrap().id(), _ => unreachable!(), } @@ -254,7 +254,7 @@ impl GraphElement for DynEntry { fn label(&self) -> Option { match self.get_type() { - EntryDataType::Id => None, + EntryDataType::Vid => None, EntryDataType::V => self .as_graph_vertex() .map(|v| v.label()) @@ -269,7 +269,7 @@ impl GraphElement for DynEntry { fn details(&self) -> Option<&DynDetails> { match self.get_type() { - EntryDataType::Id => None, + EntryDataType::Vid => None, EntryDataType::V => self .as_graph_vertex() .map(|v| v.details()) @@ -287,7 +287,7 @@ impl GraphElement for DynEntry { impl Hash for DynEntry { fn hash(&self, state: &mut H) { match self.get_type() { - EntryDataType::Id | EntryDataType::V => self.as_id().hash(state), + EntryDataType::Vid | EntryDataType::V => self.as_vid().hash(state), EntryDataType::E => self.as_graph_edge().hash(state), EntryDataType::P => self.as_graph_path().hash(state), EntryDataType::Obj => self.as_object().hash(state), @@ -308,7 +308,7 @@ impl PartialEq for DynEntry { fn eq(&self, other: &Self) -> bool { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryDataType::Id | EntryDataType::V => self.as_id().eq(&other.as_id()), + EntryDataType::Vid | EntryDataType::V => self.as_vid().eq(&other.as_vid()), EntryDataType::E => self.as_graph_edge().eq(&other.as_graph_edge()), EntryDataType::P => self.as_graph_path().eq(&other.as_graph_path()), EntryDataType::Obj => self.as_object().eq(&other.as_object()), @@ -336,7 +336,7 @@ impl PartialOrd for DynEntry { fn partial_cmp(&self, other: &Self) -> Option { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryDataType::Id | EntryDataType::V => self.as_id().partial_cmp(&other.as_id()), + EntryDataType::Vid | EntryDataType::V => self.as_vid().partial_cmp(&other.as_vid()), EntryDataType::E => self .as_graph_edge() .partial_cmp(&other.as_graph_edge()), @@ -398,10 +398,10 @@ impl_as_any!(IDEntry); impl Entry for IDEntry { fn get_type(&self) -> EntryDataType { - EntryDataType::Id + EntryDataType::Vid } - fn as_id(&self) -> Option { + fn as_vid(&self) -> Option { Some(self.id) } } @@ -448,7 +448,7 @@ impl Entry for Vertex { EntryDataType::V } - fn as_id(&self) -> Option { + fn as_vid(&self) -> Option { Some(self.id()) } @@ -462,10 +462,6 @@ impl Entry for Edge { EntryDataType::E } - fn as_id(&self) -> Option { - Some(self.id()) - } - fn as_graph_edge(&self) -> Option<&Edge> { Some(self) } @@ -476,10 +472,6 @@ impl Entry for Object { EntryDataType::Obj } - fn as_id(&self) -> Option { - None - } - fn as_object(&self) -> Option<&Object> { Some(self) } @@ -489,10 +481,6 @@ impl Entry for Intersection { fn get_type(&self) -> EntryDataType { EntryDataType::Intersect } - - fn as_id(&self) -> Option { - None - } } impl Entry for GraphPath { @@ -500,10 +488,6 @@ impl Entry for GraphPath { EntryDataType::P } - fn as_id(&self) -> Option { - None - } - fn as_graph_path(&self) -> Option<&GraphPath> { Some(self) } @@ -520,10 +504,6 @@ impl Entry for CollectionEntry { fn get_type(&self) -> EntryDataType { EntryDataType::Collection } - - fn as_id(&self) -> Option { - None - } } impl Element for CollectionEntry { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs index d304c03287b1..da4a9a92d84f 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs @@ -325,13 +325,13 @@ mod tests { use pegasus::api::{Fold, Sink}; use pegasus::result::ResultStream; use pegasus::JobConf; + use pegasus_common::downcast::AsAny; use crate::process::entry::{CollectionEntry, DynEntry, Entry}; use crate::process::operator::accum::accumulator::Accumulator; use crate::process::operator::accum::AccumFactoryGen; use crate::process::operator::tests::{init_source, init_vertex1, init_vertex2, TAG_A, TAG_B}; use crate::process::record::Record; - use pegasus_common::downcast::AsAny; fn fold_test(source: Vec, fold_opr_pb: pb::GroupBy) -> ResultStream { let conf = JobConf::new("fold_test"); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs index ccccb95382f7..083ff4f3a8d3 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs @@ -24,7 +24,7 @@ use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenError, FnGenResult}; -use crate::process::entry::{Entry, EntryDataType}; +use crate::process::entry::Entry; use crate::process::operator::flatmap::FlatMapFuncGen; use crate::process::record::{Record, RecordExpandIter, RecordPathExpandIter}; @@ -40,58 +40,52 @@ impl FlatMapFunction for EdgeExpandOperator< fn exec(&self, mut input: Record) -> FnResult { if let Some(entry) = input.get(self.start_v_tag) { - match entry.get_type() { - EntryDataType::Id | EntryDataType::V => { - let id = entry.as_id().ok_or(FnExecError::Unreachable)?; - let iter = self.stmt.exec(id)?; - match self.expand_opt { - // the case of expand edge, and get end vertex; - ExpandOpt::Vertex => { - let neighbors_iter = iter.map(|e| { - if let Some(e) = e.as_graph_edge() { - Vertex::new( - e.get_other_id(), - e.get_other_label().cloned(), - DynDetails::default(), - ) - } else { - unreachable!() - } - }); - Ok(Box::new(RecordExpandIter::new( - input, - self.alias.as_ref(), - Box::new(neighbors_iter), - ))) - } - // the case of expand neighbors, including edges/vertices - ExpandOpt::Edge => { - Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) - } - // the case of get degree. TODO: this case should be a `Map` - ExpandOpt::Degree => { - let degree = iter.count(); - input.append(object!(degree), self.alias); - Ok(Box::new(vec![input].into_iter())) - } + if let Some(id) = entry.as_vid() { + let iter = self.stmt.exec(id)?; + match self.expand_opt { + // the case of expand edge, and get end vertex; + ExpandOpt::Vertex => { + let neighbors_iter = iter.map(|e| { + if let Some(e) = e.as_graph_edge() { + Vertex::new( + e.get_other_id(), + e.get_other_label().cloned(), + DynDetails::default(), + ) + } else { + unreachable!() + } + }); + Ok(Box::new(RecordExpandIter::new( + input, + self.alias.as_ref(), + Box::new(neighbors_iter), + ))) + } + // the case of expand neighbors, including edges/vertices + ExpandOpt::Edge => { + Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) + } + // the case of get degree. TODO: this case should be a `Map` + ExpandOpt::Degree => { + let degree = iter.count(); + input.append(object!(degree), self.alias); + Ok(Box::new(vec![input].into_iter())) } } - EntryDataType::P => { - let graph_path = entry - .as_graph_path() - .ok_or(FnExecError::Unreachable)?; - let path_end = graph_path - .get_path_end() - .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; - let id = path_end.id(); - let iter = self.stmt.exec(id)?; - let curr_path = graph_path.clone(); - Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) - } - _ => Err(FnExecError::unexpected_data_error(&format!( + } else if let Some(graph_path) = entry.as_graph_path() { + let path_end = graph_path + .get_path_end() + .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; + let id = path_end.id(); + let iter = self.stmt.exec(id)?; + let curr_path = graph_path.clone(); + Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) + } else { + Err(FnExecError::unexpected_data_error(&format!( "Cannot Expand from current entry {:?}", entry - )))?, + )))? } } else { Ok(Box::new(vec![].into_iter())) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs index ada7ddc9d421..1a23bc3cd144 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs @@ -46,8 +46,8 @@ impl FilterMapFunction for AuxiliaOperator { // If queryable, then turn into graph element and do the query let graph = get_graph().ok_or(FnExecError::NullGraphError)?; let new_entry: Option = match entry.get_type() { - EntryDataType::Id | EntryDataType::V => { - let id = entry.as_id().unwrap(); + EntryDataType::Vid | EntryDataType::V => { + let id = entry.as_vid().unwrap(); let mut result_iter = graph.get_vertex(&[id], &self.query_params)?; result_iter.next().map(|mut vertex| { // TODO:confirm the update case, and avoid it if possible. @@ -68,7 +68,10 @@ impl FilterMapFunction for AuxiliaOperator { }) } EntryDataType::E => { - let id = entry.as_id().unwrap(); + let id = entry + .as_graph_edge() + .ok_or(FnExecError::Unreachable)? + .id(); let mut result_iter = graph.get_edge(&[id], &self.query_params)?; result_iter.next().map(|mut edge| { if let Some(details) = entry diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index 41a4957ee8aa..6c7b7f68ae7f 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -158,7 +158,7 @@ impl FilterMapFunction for ExpandVertexOrIntersect { "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", self.start_v_tag, input )))?; - if let Some(id) = entry.as_id() { + if let Some(id) = entry.as_vid() { let iter = self.stmt.exec(id)?; if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { // the case of expansion and intersection @@ -203,7 +203,7 @@ impl FilterMapFunction for ExpandEdgeVOrIntersect { "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", self.start_v_tag, input )))?; - if let Some(id) = entry.as_id() { + if let Some(id) = entry.as_vid() { let iter = self .stmt .exec(id)? diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs index 85361aad257c..9b43900c0142 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs @@ -50,11 +50,10 @@ impl RecordRouter { impl RouteFunction for RecordRouter { fn route(&self, t: &Record) -> FnResult { if let Some(entry) = t.get(self.shuffle_key.clone()) { - // TODO: this may occur bugs if `Id` is actually an edge entry. match entry.get_type() { - EntryDataType::Id | EntryDataType::V => { + EntryDataType::Vid | EntryDataType::V => { let id = entry - .as_id() + .as_vid() .ok_or(FnExecError::unexpected_data_error("get id failed in shuffle"))?; Ok(self.p.get_partition(&id, self.num_workers)?) } diff --git a/interactive_engine/executor/ir/runtime/src/process/record.rs b/interactive_engine/executor/ir/runtime/src/process/record.rs index 76114da7e7c4..b8f45f73be73 100644 --- a/interactive_engine/executor/ir/runtime/src/process/record.rs +++ b/interactive_engine/executor/ir/runtime/src/process/record.rs @@ -16,6 +16,7 @@ use std::borrow::BorrowMut; use std::hash::Hash; +use graph_proxy::apis::{GraphPath, Vertex}; use graph_proxy::utils::expr::eval::Context; use ir_common::{KeyId, NameOrId}; use pegasus::api::function::DynIter; @@ -23,7 +24,6 @@ use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use vec_map::VecMap; use crate::process::entry::{DynEntry, Entry, EntryDataType}; -use graph_proxy::apis::{GraphPath, Vertex}; #[derive(Debug, Clone, Default)] pub struct Record { From 342fc2bef9ae626cc966a2a4e9f6f4b226f82d97 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 6 Dec 2022 17:57:04 +0800 Subject: [PATCH 4/8] minor refine --- .../ir/integrated/tests/apply_test.rs | 8 ++-- .../ir/integrated/tests/auxilia_test.rs | 28 ++++------- .../ir/integrated/tests/expand_test.rs | 40 +++++++--------- .../ir/integrated/tests/graph_query_test.rs | 2 +- .../ir/integrated/tests/match_test.rs | 35 +++----------- .../executor/ir/integrated/tests/scan_test.rs | 18 +++---- .../executor/ir/runtime/src/process/entry.rs | 47 ++++++++++--------- .../src/process/operator/accum/accum.rs | 14 +++--- .../src/process/operator/filter/select.rs | 12 ++--- .../process/operator/flatmap/edge_expand.rs | 2 +- .../src/process/operator/flatmap/get_v.rs | 2 +- .../src/process/operator/flatmap/unfold.rs | 4 +- .../runtime/src/process/operator/join/join.rs | 2 +- .../src/process/operator/keyed/keyed.rs | 2 +- .../src/process/operator/map/auxilia.rs | 6 +-- .../runtime/src/process/operator/map/get_v.rs | 2 +- .../src/process/operator/map/path_start.rs | 2 +- .../src/process/operator/map/project.rs | 6 +-- .../ir/runtime/src/process/operator/mod.rs | 2 +- .../runtime/src/process/operator/shuffle.rs | 2 +- .../runtime/src/process/operator/sink/sink.rs | 4 +- .../process/operator/sink/sink_vineyard.rs | 8 ++-- .../runtime/src/process/operator/sort/sort.rs | 20 +++----- 23 files changed, 113 insertions(+), 155 deletions(-) diff --git a/interactive_engine/executor/ir/integrated/tests/apply_test.rs b/interactive_engine/executor/ir/integrated/tests/apply_test.rs index 66f16a381732..6ecf7b99e4e0 100644 --- a/interactive_engine/executor/ir/integrated/tests/apply_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/apply_test.rs @@ -90,7 +90,7 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { + if let Some(vertex) = record.get(None).unwrap().as_vertex() { result_collection.push(vertex.id() as DefaultId); } } @@ -127,7 +127,7 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { + if let Some(vertex) = record.get(None).unwrap().as_vertex() { result_collection.push(vertex.id() as DefaultId); } } @@ -229,7 +229,7 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { + if let Some(vertex) = record.get(None).unwrap().as_vertex() { let cnt = record .get(Some(TAG_A)) .unwrap() @@ -276,7 +276,7 @@ mod test { match result { Ok(res) => { let record = parse_result(res).unwrap(); - if let Some(vertex) = record.get(None).unwrap().as_graph_vertex() { + if let Some(vertex) = record.get(None).unwrap().as_vertex() { let object = record .get(Some(TAG_A)) .unwrap() diff --git a/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs b/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs index 104683c85e10..0fae3d5d73f2 100644 --- a/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/auxilia_test.rs @@ -85,11 +85,7 @@ mod test { let expected_ids = vec![2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_A)).unwrap().as_vertex() { result_ids.push(element.id()); } } @@ -133,7 +129,7 @@ mod test { let expected_ids_with_prop = vec![(2, "vadas".to_string().into()), (4, "josh".to_string().into())]; let mut result_ids_with_prop = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids_with_prop.push(( element.id(), element @@ -186,11 +182,7 @@ mod test { let expected_ids_with_prop = vec![(2, "vadas".to_string().into()), (4, "josh".to_string().into())]; let mut result_ids_with_prop = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_A)).unwrap().as_vertex() { result_ids_with_prop.push(( element.id(), element @@ -247,7 +239,7 @@ mod test { let expected_ids_with_prop = vec![(2, "vadas".to_string().into())]; let mut result_ids_with_prop = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids_with_prop.push(( element.id(), element @@ -304,11 +296,7 @@ mod test { let expected_ids = vec![2]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_A)).unwrap().as_vertex() { result_ids.push(element.id()); } } @@ -358,7 +346,7 @@ mod test { let mut results: Vec<(Object, Object)> = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { let details = element.details().unwrap(); results.push(( details @@ -414,7 +402,7 @@ mod test { let mut results: Vec = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { let details = element.details().unwrap(); results.push( details @@ -472,7 +460,7 @@ mod test { let mut results: Vec = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { let details = element.details().unwrap(); results.push( details diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index 39e9beed9cea..db5d6862aeeb 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -151,7 +151,7 @@ mod test { let v5: DefaultId = LDBCVertexParser::to_global_id(5, 1); let mut expected_ids = vec![v2, v3, v3, v3, v4, v5]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -178,7 +178,7 @@ mod test { let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); let expected_edges = vec![(v1, v2), (v1, v4)]; while let Some(Ok(record)) = result.next() { - if let Some(e) = record.get(None).unwrap().as_graph_edge() { + if let Some(e) = record.get(None).unwrap().as_edge() { result_edges.push((e.src_id as usize, e.dst_id as usize)); } } @@ -208,7 +208,7 @@ mod test { let mut expected_edges = vec![(v1, v2), (v1, v3), (v1, v4), (v4, v3), (v4, v5), (v6, v3)]; expected_edges.sort(); while let Some(Ok(record)) = result.next() { - if let Some(e) = record.get(None).unwrap().as_graph_edge() { + if let Some(e) = record.get(None).unwrap().as_edge() { result_edges.push((e.src_id as usize, e.dst_id as usize)); } } @@ -232,7 +232,7 @@ mod test { let v1: DefaultId = LDBCVertexParser::to_global_id(1, 0); let expected_dst_ids_with_prop = vec![(v1, object!(0.5)), (v1, object!(1.0))]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_edge() { + if let Some(element) = record.get(None).unwrap().as_edge() { result_ids_with_prop.push(( element.get_other_id() as usize, element @@ -286,11 +286,7 @@ mod test { let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); let mut expected_ids = vec![v2, v4]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_B)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_B)).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -338,7 +334,7 @@ mod test { let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); let mut expected_ids = vec![v2, v4]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -381,7 +377,7 @@ mod test { let v2: DefaultId = LDBCVertexParser::to_global_id(2, 0); let expected_ids = vec![v2]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -405,7 +401,7 @@ mod test { let v2: DefaultId = LDBCVertexParser::to_global_id(2, 0); let expected_ids = vec![v2]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -448,7 +444,7 @@ mod test { let expected_ids = vec![2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); assert!(element .details() @@ -497,7 +493,7 @@ mod test { let expected_ids = vec![1, 4, 4, 6]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); assert!(element .details() @@ -546,7 +542,7 @@ mod test { let expected_ids = vec![1, 1, 2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); assert!(element .details() @@ -595,7 +591,7 @@ mod test { let expected_ids = vec![1, 1, 2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); assert!(element .details() @@ -628,7 +624,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_results = vec![(v1, 3), (v2, 0), (v3, 0), (v4, 2), (v5, 0), (v6, 1)]; while let Some(Ok(record)) = pegasus_result.next() { - if let Some(v) = record.get(None).unwrap().as_graph_vertex() { + if let Some(v) = record.get(None).unwrap().as_vertex() { if let Some(degree_obj) = record.get(Some(1)).unwrap().as_object() { results.push((v.id() as DefaultId, degree_obj.as_u64().unwrap())); } @@ -660,7 +656,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_results = vec![(v1, 0), (v2, 1), (v3, 3), (v4, 1), (v5, 1), (v6, 0)]; while let Some(Ok(record)) = pegasus_result.next() { - if let Some(v) = record.get(None).unwrap().as_graph_vertex() { + if let Some(v) = record.get(None).unwrap().as_vertex() { if let Some(degree_obj) = record.get(Some(1)).unwrap().as_object() { results.push((v.id() as DefaultId, degree_obj.as_u64().unwrap())); } @@ -692,7 +688,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_results = vec![(v1, 3), (v2, 1), (v3, 3), (v4, 3), (v5, 1), (v6, 1)]; while let Some(Ok(record)) = pegasus_result.next() { - if let Some(v) = record.get(None).unwrap().as_graph_vertex() { + if let Some(v) = record.get(None).unwrap().as_vertex() { if let Some(degree_obj) = record.get(Some(1)).unwrap().as_object() { results.push((v.id() as DefaultId, degree_obj.as_u64().unwrap())); } @@ -916,7 +912,7 @@ mod test { let expected_ids = vec![v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); } } @@ -984,7 +980,7 @@ mod test { let mut expected_ids = vec![v1, v1, v3, v3, v4, v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); } } @@ -1062,7 +1058,7 @@ mod test { let mut expected_ids = vec![v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize); } } diff --git a/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs b/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs index b2059a67625a..cb5c2a3f0440 100644 --- a/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/graph_query_test.rs @@ -75,7 +75,7 @@ mod test { match result { Ok(res) => { let entry = parse_result(res).unwrap(); - if let Some(vertex) = entry.get(None).unwrap().as_graph_vertex() { + if let Some(vertex) = entry.get(None).unwrap().as_vertex() { result_collection.push(vertex.id()); } } diff --git a/interactive_engine/executor/ir/integrated/tests/match_test.rs b/interactive_engine/executor/ir/integrated/tests/match_test.rs index 33c40b82789f..12c11c5b1587 100644 --- a/interactive_engine/executor/ir/integrated/tests/match_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/match_test.rs @@ -231,18 +231,9 @@ mod test { match result { Ok(res) => { let entry = parse_result(res).unwrap(); - let a = entry - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex(); - let b = entry - .get(Some(TAG_B)) - .unwrap() - .as_graph_vertex(); - let c = entry - .get(Some(TAG_C)) - .unwrap() - .as_graph_vertex(); + let a = entry.get(Some(TAG_A)).unwrap().as_vertex(); + let b = entry.get(Some(TAG_B)).unwrap().as_vertex(); + let c = entry.get(Some(TAG_C)).unwrap().as_vertex(); result_collection.push((a.unwrap().id(), b.unwrap().id(), c.unwrap().id())); } Err(e) => { @@ -280,22 +271,10 @@ mod test { match result { Ok(res) => { let entry = parse_result(res).unwrap(); - let a = entry - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex(); - let b = entry - .get(Some(TAG_B)) - .unwrap() - .as_graph_vertex(); - let c = entry - .get(Some(TAG_C)) - .unwrap() - .as_graph_vertex(); - let d = entry - .get(Some(TAG_D)) - .unwrap() - .as_graph_vertex(); + let a = entry.get(Some(TAG_A)).unwrap().as_vertex(); + let b = entry.get(Some(TAG_B)).unwrap().as_vertex(); + let c = entry.get(Some(TAG_C)).unwrap().as_vertex(); + let d = entry.get(Some(TAG_D)).unwrap().as_vertex(); result_collection.push(( a.unwrap().id(), b.unwrap().id(), diff --git a/interactive_engine/executor/ir/integrated/tests/scan_test.rs b/interactive_engine/executor/ir/integrated/tests/scan_test.rs index d6a5b73e26c5..2a4aa82af49d 100644 --- a/interactive_engine/executor/ir/integrated/tests/scan_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/scan_test.rs @@ -55,7 +55,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_ids = vec![v1, v2, v3, v4, v5, v6]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -80,7 +80,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_ids = vec![v1, v2, v4, v6]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -107,7 +107,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_ids = vec![v1, v2, v3, v4, v5, v6]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -130,7 +130,7 @@ mod test { let v1: DefaultId = LDBCVertexParser::to_global_id(1, 0); let mut expected_ids = vec![v1]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -154,7 +154,7 @@ mod test { let v2: DefaultId = LDBCVertexParser::to_global_id(2, 0); let mut expected_ids = vec![v1, v2]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id() as usize) } } @@ -172,7 +172,7 @@ mod test { scan_gen(pb::Scan { scan_opt: 0, alias: None, params: Some(params), idx_predicate: None }); let mut result_count = 0; for record in source_iter { - if let Some(_element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(_element) = record.get(None).unwrap().as_vertex() { result_count += 1 } } @@ -194,7 +194,7 @@ mod test { let v6: DefaultId = LDBCVertexParser::to_global_id(6, 0); let mut expected_ids = vec![(v1, v2), (v1, v3), (v1, v4), (v4, v3), (v4, v5), (v6, v3)]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_edge() { + if let Some(element) = record.get(None).unwrap().as_edge() { result_ids.push((element.src_id as usize, element.dst_id as usize)) } } @@ -218,7 +218,7 @@ mod test { let v4: DefaultId = LDBCVertexParser::to_global_id(4, 0); let mut expected_ids = vec![(v1, v2), (v1, v4)]; for record in source_iter { - if let Some(element) = record.get(None).unwrap().as_graph_edge() { + if let Some(element) = record.get(None).unwrap().as_edge() { result_ids.push((element.src_id as usize, element.dst_id as usize)) } } @@ -236,7 +236,7 @@ mod test { scan_gen(pb::Scan { scan_opt: 1, alias: None, params: Some(params), idx_predicate: None }); let mut result_count = 0; for record in source_iter { - if let Some(_element) = record.get(None).unwrap().as_graph_edge() { + if let Some(_element) = record.get(None).unwrap().as_edge() { result_count += 1 } } diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs index 63d1c11babe6..0dbb9a8c4e47 100644 --- a/interactive_engine/executor/ir/runtime/src/process/entry.rs +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -66,10 +66,10 @@ pub trait Entry: Debug + Send + Sync + AsAny + Element { fn as_vid(&self) -> Option { None } - fn as_graph_vertex(&self) -> Option<&Vertex> { + fn as_vertex(&self) -> Option<&Vertex> { None } - fn as_graph_edge(&self) -> Option<&Edge> { + fn as_edge(&self) -> Option<&Edge> { None } fn as_graph_path(&self) -> Option<&GraphPath> { @@ -128,12 +128,12 @@ impl Entry for DynEntry { self.inner.as_vid() } - fn as_graph_vertex(&self) -> Option<&Vertex> { - self.inner.as_graph_vertex() + fn as_vertex(&self) -> Option<&Vertex> { + self.inner.as_vertex() } - fn as_graph_edge(&self) -> Option<&Edge> { - self.inner.as_graph_edge() + fn as_edge(&self) -> Option<&Edge> { + self.inner.as_edge() } fn as_graph_path(&self) -> Option<&GraphPath> { @@ -155,13 +155,11 @@ impl Encode for DynEntry { } EntryDataType::V => { writer.write_u8(1)?; - self.as_graph_vertex() - .unwrap() - .write_to(writer)?; + self.as_vertex().unwrap().write_to(writer)?; } EntryDataType::E => { writer.write_u8(2)?; - self.as_graph_edge().unwrap().write_to(writer)?; + self.as_edge().unwrap().write_to(writer)?; } EntryDataType::P => { writer.write_u8(3)?; @@ -247,7 +245,7 @@ impl GraphElement for DynEntry { fn id(&self) -> ID { match self.get_type() { EntryDataType::Vid | EntryDataType::V => self.as_vid().unwrap(), - EntryDataType::E => self.as_graph_edge().unwrap().id(), + EntryDataType::E => self.as_edge().unwrap().id(), _ => unreachable!(), } } @@ -256,11 +254,11 @@ impl GraphElement for DynEntry { match self.get_type() { EntryDataType::Vid => None, EntryDataType::V => self - .as_graph_vertex() + .as_vertex() .map(|v| v.label()) .unwrap_or(None), EntryDataType::E => self - .as_graph_edge() + .as_edge() .map(|v| v.label()) .unwrap_or(None), _ => unreachable!(), @@ -271,11 +269,11 @@ impl GraphElement for DynEntry { match self.get_type() { EntryDataType::Vid => None, EntryDataType::V => self - .as_graph_vertex() + .as_vertex() .map(|v| v.details()) .unwrap_or(None), EntryDataType::E => self - .as_graph_edge() + .as_edge() .map(|v| v.details()) .unwrap_or(None), _ => unreachable!(), @@ -288,7 +286,7 @@ impl Hash for DynEntry { fn hash(&self, state: &mut H) { match self.get_type() { EntryDataType::Vid | EntryDataType::V => self.as_vid().hash(state), - EntryDataType::E => self.as_graph_edge().hash(state), + EntryDataType::E => self.as_edge().hash(state), EntryDataType::P => self.as_graph_path().hash(state), EntryDataType::Obj => self.as_object().hash(state), EntryDataType::Intersect => self @@ -309,7 +307,7 @@ impl PartialEq for DynEntry { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { EntryDataType::Vid | EntryDataType::V => self.as_vid().eq(&other.as_vid()), - EntryDataType::E => self.as_graph_edge().eq(&other.as_graph_edge()), + EntryDataType::E => self.as_edge().eq(&other.as_edge()), EntryDataType::P => self.as_graph_path().eq(&other.as_graph_path()), EntryDataType::Obj => self.as_object().eq(&other.as_object()), EntryDataType::Intersect => self @@ -337,9 +335,7 @@ impl PartialOrd for DynEntry { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { EntryDataType::Vid | EntryDataType::V => self.as_vid().partial_cmp(&other.as_vid()), - EntryDataType::E => self - .as_graph_edge() - .partial_cmp(&other.as_graph_edge()), + EntryDataType::E => self.as_edge().partial_cmp(&other.as_edge()), EntryDataType::P => self .as_graph_path() .partial_cmp(&other.as_graph_path()), @@ -452,7 +448,7 @@ impl Entry for Vertex { Some(self.id()) } - fn as_graph_vertex(&self) -> Option<&Vertex> { + fn as_vertex(&self) -> Option<&Vertex> { Some(self) } } @@ -462,7 +458,7 @@ impl Entry for Edge { EntryDataType::E } - fn as_graph_edge(&self) -> Option<&Edge> { + fn as_edge(&self) -> Option<&Edge> { Some(self) } } @@ -609,6 +605,13 @@ impl From for DynEntry { } } +impl From> for DynEntry { + fn from(vec: Vec) -> Self { + let c = CollectionEntry { inner: vec }; + DynEntry::new(c) + } +} + impl From for DynEntry { fn from(c: CollectionEntry) -> Self { DynEntry::new(c) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs index da4a9a92d84f..9d42f78e3cb8 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs @@ -25,7 +25,7 @@ use ir_common::KeyId; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use crate::error::{FnExecError, FnExecResult, FnGenError, FnGenResult}; -use crate::process::entry::{CollectionEntry, DynEntry, Entry}; +use crate::process::entry::{DynEntry, Entry}; use crate::process::operator::accum::accumulator::{ Accumulator, Count, DistinctCount, Maximum, Minimum, Sum, ToList, ToSet, }; @@ -98,11 +98,11 @@ impl Accumulator for EntryAccumulator { match self { EntryAccumulator::ToCount(count) => { let cnt = count.finalize()?; - Ok(DynEntry::new(object!(cnt))) + Ok(object!(cnt).into()) } EntryAccumulator::ToList(list) => { let list_entry: Vec = list.finalize()?; - Ok(DynEntry::new(CollectionEntry { inner: list_entry })) + Ok(list_entry.into()) } EntryAccumulator::ToMin(min) => min .finalize()? @@ -112,11 +112,11 @@ impl Accumulator for EntryAccumulator { .ok_or(FnExecError::accum_error("max_entry is none")), EntryAccumulator::ToSet(set) => { let set_entry = set.finalize()?; - Ok(DynEntry::new(CollectionEntry { inner: set_entry })) + Ok(set_entry.into()) } EntryAccumulator::ToDistinctCount(distinct_count) => { let cnt = distinct_count.finalize()?; - Ok(DynEntry::new(object!(cnt))) + Ok(object!(cnt).into()) } EntryAccumulator::ToSum(sum) => sum .finalize()? @@ -127,7 +127,7 @@ impl Accumulator for EntryAccumulator { .ok_or(FnExecError::accum_error("sum_entry is none"))?; let cnt = count.finalize()?; // TODO: confirm if it should be Object::None, or throw error; - let result = DynEntry::new(Object::None); + let result = (Object::None).into(); if cnt == 0 { warn!("cnt value is 0 in accum avg"); Ok(result) @@ -144,7 +144,7 @@ impl Accumulator for EntryAccumulator { .map(|val| val.div(primitive_cnt)) .map_err(|e| FnExecError::accum_error(&format!("{}", e)))?; let result_obj: Object = result.into(); - Ok(DynEntry::new(result_obj)) + Ok(result_obj.into()) } } } else { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs b/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs index 6865cc883d32..09a26c9f37d3 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/filter/select.rs @@ -117,7 +117,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { assert!(element.id() < 2) } count += 1; @@ -216,7 +216,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { assert!(element.id() > 1) } count += 1; @@ -232,7 +232,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { assert!(element.id() < 2) } count += 1; @@ -249,7 +249,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { { assert_eq!( element @@ -276,7 +276,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { { assert_eq!(element.id(), 1); } @@ -294,7 +294,7 @@ mod tests { let mut result = select_test(init_source(), select_opr_pb); let mut count = 0; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { { assert_eq!(element.label().unwrap().clone(), PERSON_LABEL); } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs index 083ff4f3a8d3..a1b5ad725425 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs @@ -46,7 +46,7 @@ impl FlatMapFunction for EdgeExpandOperator< // the case of expand edge, and get end vertex; ExpandOpt::Vertex => { let neighbors_iter = iter.map(|e| { - if let Some(e) = e.as_graph_edge() { + if let Some(e) = e.as_edge() { Vertex::new( e.get_other_id(), e.get_other_label().cloned(), diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs index 7816770ced35..cfcfc5575f82 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/get_v.rs @@ -38,7 +38,7 @@ impl FlatMapFunction for GetBothVOperator { fn exec(&self, input: Record) -> FnResult { if let Some(entry) = input.get(self.start_tag) { - if let Some(e) = entry.as_graph_edge() { + if let Some(e) = entry.as_edge() { let src_vertex = Vertex::new(e.src_id, e.get_src_label().map(|l| l.clone()), DynDetails::default()); let dst_vertex = diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs index 2947435375b0..a0ec388fa91e 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs @@ -171,7 +171,7 @@ mod tests { let expected_result = vec![1, 2]; let mut result_ids = vec![]; while let Some(Ok(res)) = result.next() { - if let Some(v) = res.get(None).unwrap().as_graph_vertex() { + if let Some(v) = res.get(None).unwrap().as_vertex() { result_ids.push(v.id()); } } @@ -194,7 +194,7 @@ mod tests { let expected_result = vec![1, 2]; let mut result_ids = vec![]; while let Some(Ok(res)) = result.next() { - if let Some(v) = res.get(None).unwrap().as_graph_vertex() { + if let Some(v) = res.get(None).unwrap().as_vertex() { result_ids.push(v.id()); } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs b/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs index 30dedf558838..bf6cae5b5efc 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/join/join.rs @@ -155,7 +155,7 @@ mod tests { let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id()); } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs b/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs index 618303f79467..69327acf12f5 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/keyed/keyed.rs @@ -130,7 +130,7 @@ mod tests { let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id()); } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs index 1a23bc3cd144..179e65867daf 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs @@ -52,7 +52,7 @@ impl FilterMapFunction for AuxiliaOperator { result_iter.next().map(|mut vertex| { // TODO:confirm the update case, and avoid it if possible. if let Some(details) = entry - .as_graph_vertex() + .as_vertex() .map(|v| v.details()) .unwrap_or(None) { @@ -69,13 +69,13 @@ impl FilterMapFunction for AuxiliaOperator { } EntryDataType::E => { let id = entry - .as_graph_edge() + .as_edge() .ok_or(FnExecError::Unreachable)? .id(); let mut result_iter = graph.get_edge(&[id], &self.query_params)?; result_iter.next().map(|mut edge| { if let Some(details) = entry - .as_graph_edge() + .as_edge() .map(|e| e.details()) .unwrap_or(None) { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs index 55bb75b2719b..19033c6436bd 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/get_v.rs @@ -37,7 +37,7 @@ struct GetVertexOperator { impl FilterMapFunction for GetVertexOperator { fn exec(&self, mut input: Record) -> FnResult> { if let Some(entry) = input.get(self.start_tag) { - if let Some(e) = entry.as_graph_edge() { + if let Some(e) = entry.as_edge() { let (id, label) = match self.opt { VOpt::Start => (e.src_id, e.get_src_label()), VOpt::End => (e.dst_id, e.get_dst_label()), diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs index b8caeadda7d2..3df72868ff4a 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/path_start.rs @@ -38,7 +38,7 @@ impl FilterMapFunction for PathStartOperator { fn exec(&self, mut input: Record) -> FnResult> { if let Some(entry) = input.get(self.start_tag) { let v = entry - .as_graph_vertex() + .as_vertex() .ok_or(FnExecError::unexpected_data_error(&format!( "tag {:?} does not refer to a graph vertex element in record {:?}", self.start_tag, input diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs index b332378005a3..4799b34244f9 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs @@ -418,7 +418,7 @@ mod tests { let v = res .get(Some(TAG_A)) .unwrap() - .as_graph_vertex() + .as_vertex() .unwrap(); a_results.push(v.id()); let b_val = res @@ -638,12 +638,12 @@ mod tests { let v1 = res .get(Some(TAG_C)) .unwrap() - .as_graph_vertex() + .as_vertex() .unwrap(); let v2 = res .get(Some(TAG_D)) .unwrap() - .as_graph_vertex() + .as_vertex() .unwrap(); results.push(v1.id()); results.push(v2.id()); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs b/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs index 3faf217c80dc..6cfe0aff94ac 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/mod.rs @@ -308,7 +308,7 @@ pub(crate) mod tests { let expected = init_vertex2(); let record = init_record(); let entry = tag_key.get_arc_entry(&record).unwrap(); - if let Some(element) = entry.as_graph_vertex() { + if let Some(element) = entry.as_vertex() { assert_eq!(element.id(), expected.id()); } else { assert!(false); diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs index 9b43900c0142..b1c6af44dd8e 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs @@ -59,7 +59,7 @@ impl RouteFunction for RecordRouter { } EntryDataType::E => { let e = entry - .as_graph_edge() + .as_edge() .ok_or(FnExecError::unexpected_data_error("get edge failed in shuffle"))?; Ok(self .p diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs index ca3add9d8a22..341b3d05becd 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs @@ -86,11 +86,11 @@ impl RecordSinkEncoder { fn element_to_pb(&self, e: &DynEntry) -> result_pb::Element { let inner = match e.get_type() { EntryDataType::V => { - let vertex_pb = self.vertex_to_pb(e.as_graph_vertex().unwrap()); + let vertex_pb = self.vertex_to_pb(e.as_vertex().unwrap()); Some(result_pb::element::Inner::Vertex(vertex_pb)) } EntryDataType::E => { - let edge_pb = self.edge_to_pb(e.as_graph_edge().unwrap()); + let edge_pb = self.edge_to_pb(e.as_edge().unwrap()); Some(result_pb::element::Inner::Edge(edge_pb)) } EntryDataType::P => { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs index 033743ee0c45..e6848a2b6bbc 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink_vineyard.rs @@ -46,7 +46,7 @@ impl Accumulator for GraphSinkEncoder { "tag {:?} in GraphWriter on {:?}", sink_key, next )))?; - if let Some(v) = entry.as_graph_vertex() { + if let Some(v) = entry.as_vertex() { let vertex_pk = graph .get_primary_key(&v.id())? .ok_or(GraphProxyError::query_store_error("get_primary_key() returns empty pk"))?; @@ -62,7 +62,7 @@ impl Accumulator for GraphSinkEncoder { break; } } - } else if let Some(e) = entry.as_graph_edge() { + } else if let Some(e) = entry.as_edge() { let src_vertex_pk = graph .get_primary_key(&e.src_id)? @@ -267,10 +267,10 @@ mod tests { impl Accumulator for TestGraphWriter { fn accum(&mut self, mut next: Record) -> FnExecResult<()> { let entry = next.take(None).unwrap(); - if let Some(v) = entry.as_graph_vertex() { + if let Some(v) = entry.as_vertex() { let pk = get_primary_key(&v.id()); self.add_vertex(v.label().unwrap().clone(), pk, v.details().cloned())?; - } else if let Some(e) = entry.as_graph_edge() { + } else if let Some(e) = entry.as_edge() { let src_pk = get_primary_key(&e.src_id); let dst_pk = get_primary_key(&e.dst_id); self.add_edge( diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs index 3b98e5822a65..e7c2015ded1c 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sort/sort.rs @@ -129,7 +129,7 @@ mod tests { let mut result = sort_test(init_source(), sort_opr); let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id()); } } @@ -150,7 +150,7 @@ mod tests { let mut result = sort_test(init_source(), sort_opr); let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_ids.push(element.id()); } } @@ -171,7 +171,7 @@ mod tests { let mut result = sort_test(init_source(), sort_opr); let mut result_name = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { result_name.push( element .details() @@ -214,7 +214,7 @@ mod tests { let mut result = sort_test(source, sort_opr); let mut result_name_ages = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_graph_vertex() { + if let Some(element) = record.get(None).unwrap().as_vertex() { let details = element.details().unwrap(); result_name_ages.push(( details @@ -251,11 +251,7 @@ mod tests { let mut result = sort_test(init_source_with_tag(), sort_opr); let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_A)).unwrap().as_vertex() { result_ids.push(element.id()); } } @@ -276,11 +272,7 @@ mod tests { let mut result = sort_test(init_source_with_tag(), sort_opr); let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record - .get(Some(TAG_A)) - .unwrap() - .as_graph_vertex() - { + if let Some(element) = record.get(Some(TAG_A)).unwrap().as_vertex() { result_ids.push(element.id()); } } From dbf83dc18fc4eea30b0fbdc38388cd42c2cef897 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Mon, 19 Dec 2022 16:53:26 +0800 Subject: [PATCH 5/8] [GIE/Runtime] refine the codes, and remove `Add` Trait for `DynEntry` --- .../ir/integrated/tests/expand_test.rs | 6 +- .../executor/ir/runtime/src/process/entry.rs | 66 ++++++++----------- .../src/process/operator/accum/accum.rs | 65 ++++++++++-------- .../src/process/operator/flatmap/unfold.rs | 4 +- .../process/operator/map/expand_intersect.rs | 46 ++++++------- .../runtime/src/process/operator/map/mod.rs | 2 +- .../runtime/src/process/operator/sink/sink.rs | 4 +- 7 files changed, 97 insertions(+), 96 deletions(-) diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index db5d6862aeeb..13084c4e9438 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -35,7 +35,7 @@ mod test { use pegasus_common::downcast::AsAny; use runtime::process::entry::Entry; use runtime::process::operator::flatmap::FlatMapFuncGen; - use runtime::process::operator::map::{FilterMapFuncGen, Intersection}; + use runtime::process::operator::map::{FilterMapFuncGen, IntersectionEntry}; use runtime::process::operator::source::SourceOperator; use runtime::process::record::Record; @@ -756,7 +756,7 @@ mod test { .get(Some(TAG_C)) .unwrap() .as_any_ref() - .downcast_ref::() + .downcast_ref::() .unwrap(); let mut result_collection: Vec = intersection .clone() @@ -833,7 +833,7 @@ mod test { .get(Some(TAG_C)) .unwrap() .as_any_ref() - .downcast_ref::() + .downcast_ref::() .unwrap(); let mut result_collection: Vec = intersection diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs index 0dbb9a8c4e47..80f1b270740b 100644 --- a/interactive_engine/executor/ir/runtime/src/process/entry.rs +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -31,16 +31,24 @@ use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use pegasus_common::downcast::*; use pegasus_common::impl_as_any; -use crate::process::operator::map::Intersection; +use crate::process::operator::map::IntersectionEntry; #[derive(Debug)] pub enum EntryDataType { - Vid, // A vertex global id + // TODO: Specified as a vertex global id for tmp; + // After we separate GetVertexProperty and GetEdgeProperty in `Auxilia`, this would be an `Id`. + Vid, + /// Graph Vertex V, + /// Graph Edge E, + /// Graph Path P, + /// Common data type of `Object`, including Primitives, String, etc. Obj, + /// A specific type used in `ExtendIntersect`, for an optimized implementation of `Intersection` Intersect, + /// Type of collection consisting of entries Collection, } @@ -63,6 +71,7 @@ impl PartialEq for EntryDataType { pub trait Entry: Debug + Send + Sync + AsAny + Element { fn get_type(&self) -> EntryDataType; + // TODO: would be `as_id()` fn as_vid(&self) -> Option { None } @@ -173,7 +182,7 @@ impl Encode for DynEntry { writer.write_u8(5)?; self.inner .as_any_ref() - .downcast_ref::() + .downcast_ref::() .unwrap() .write_to(writer)?; } @@ -196,7 +205,7 @@ impl Decode for DynEntry { match entry_type { 0 => { let id = read_id(reader)?; - Ok(DynEntry::new(IDEntry { id })) + Ok(DynEntry::new(IdEntry { id })) } 1 => { let vertex = Vertex::read_from(reader)?; @@ -215,7 +224,7 @@ impl Decode for DynEntry { Ok(DynEntry::new(obj)) } 5 => { - let intersect = Intersection::read_from(reader)?; + let intersect = IntersectionEntry::read_from(reader)?; Ok(DynEntry::new(intersect)) } 6 => { @@ -291,7 +300,7 @@ impl Hash for DynEntry { EntryDataType::Obj => self.as_object().hash(state), EntryDataType::Intersect => self .as_any_ref() - .downcast_ref::() + .downcast_ref::() .hash(state), EntryDataType::Collection => self .as_any_ref() @@ -312,10 +321,10 @@ impl PartialEq for DynEntry { EntryDataType::Obj => self.as_object().eq(&other.as_object()), EntryDataType::Intersect => self .as_any_ref() - .downcast_ref::() + .downcast_ref::() .eq(&other .as_any_ref() - .downcast_ref::()), + .downcast_ref::()), EntryDataType::Collection => self .as_any_ref() .downcast_ref::() @@ -342,11 +351,11 @@ impl PartialOrd for DynEntry { EntryDataType::Obj => self.as_object().partial_cmp(&other.as_object()), EntryDataType::Intersect => self .as_any_ref() - .downcast_ref::() + .downcast_ref::() .partial_cmp( &other .as_any_ref() - .downcast_ref::(), + .downcast_ref::(), ), EntryDataType::Collection => self .as_any_ref() @@ -366,33 +375,14 @@ impl PartialOrd for DynEntry { // demanded when need to group (ToSet) the entry; impl Eq for DynEntry {} -// demanded when need to group (ToSum) the entry; -impl std::ops::Add for DynEntry { - type Output = DynEntry; - - fn add(self, rhs: Self) -> Self::Output { - if (self.get_type()).eq(&rhs.get_type()) { - if EntryDataType::Obj.eq(&self.get_type()) { - match (self.as_object(), rhs.as_object()) { - (Some(Object::Primitive(p1)), Some(Object::Primitive(p2))) => { - return DynEntry::new(Object::Primitive(p1.add(p2.clone()))); - } - _ => {} - } - } - } - DynEntry::new(Object::None) - } -} - #[derive(Debug, Clone, Default)] -pub struct IDEntry { +pub struct IdEntry { id: ID, } -impl_as_any!(IDEntry); +impl_as_any!(IdEntry); -impl Entry for IDEntry { +impl Entry for IdEntry { fn get_type(&self) -> EntryDataType { EntryDataType::Vid } @@ -402,20 +392,20 @@ impl Entry for IDEntry { } } -impl Encode for IDEntry { +impl Encode for IdEntry { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { write_id(writer, self.id) } } -impl Decode for IDEntry { +impl Decode for IdEntry { fn read_from(reader: &mut R) -> std::io::Result { let id = read_id(reader)?; - Ok(IDEntry { id }) + Ok(IdEntry { id }) } } -impl Element for IDEntry { +impl Element for IdEntry { fn as_graph_element(&self) -> Option<&dyn GraphElement> { Some(self) } @@ -429,7 +419,7 @@ impl Element for IDEntry { } } -impl GraphElement for IDEntry { +impl GraphElement for IdEntry { fn id(&self) -> ID { self.id } @@ -473,7 +463,7 @@ impl Entry for Object { } } -impl Entry for Intersection { +impl Entry for IntersectionEntry { fn get_type(&self) -> EntryDataType { EntryDataType::Intersect } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs index 9d42f78e3cb8..d320d4969b76 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs @@ -41,8 +41,8 @@ pub enum EntryAccumulator { ToMax(Maximum), ToSet(ToSet), ToDistinctCount(DistinctCount), - ToSum(Sum), - ToAvg(Sum, Count<()>), + ToSum(Sum), + ToAvg(Sum, Count<()>), } /// Accumulator for Record, including multiple accumulators for entries(columns) in Record. @@ -83,9 +83,31 @@ impl Accumulator for EntryAccumulator { EntryAccumulator::ToMax(max) => max.accum(next), EntryAccumulator::ToSet(set) => set.accum(next), EntryAccumulator::ToDistinctCount(distinct_count) => distinct_count.accum(next), - EntryAccumulator::ToSum(sum) => sum.accum(next), + EntryAccumulator::ToSum(sum) => { + let primitive = next + .as_object() + .ok_or(FnExecError::unexpected_data_error("DynEntry is not a object type `Sum`"))? + .as_primitive() + .map_err(|e| { + FnExecError::unexpected_data_error(&format!( + "DynEntry is not a primitive type `Sum` {}", + e + )) + })?; + sum.accum(primitive) + } EntryAccumulator::ToAvg(sum, count) => { - sum.accum(next)?; + let primitive = next + .as_object() + .ok_or(FnExecError::unexpected_data_error("DynEntry is not a object type `ToAvg`"))? + .as_primitive() + .map_err(|e| { + FnExecError::unexpected_data_error(&format!( + "DynEntry is not a primitive type `ToAvg` {}", + e + )) + })?; + sum.accum(primitive)?; count.accum(()) } } @@ -118,11 +140,14 @@ impl Accumulator for EntryAccumulator { let cnt = distinct_count.finalize()?; Ok(object!(cnt).into()) } - EntryAccumulator::ToSum(sum) => sum - .finalize()? - .ok_or(FnExecError::accum_error("sum_entry is none")), + EntryAccumulator::ToSum(sum) => { + let primitive = sum + .finalize()? + .ok_or(FnExecError::accum_error("sum_entry is none"))?; + Ok(object!(primitive).into()) + } EntryAccumulator::ToAvg(sum, count) => { - let sum_entry = sum + let sum_primitive = sum .finalize()? .ok_or(FnExecError::accum_error("sum_entry is none"))?; let cnt = count.finalize()?; @@ -131,24 +156,10 @@ impl Accumulator for EntryAccumulator { if cnt == 0 { warn!("cnt value is 0 in accum avg"); Ok(result) - } else if let Some(sum_val) = sum_entry.as_object() { - match sum_val { - Object::None => { - warn!("sum value is none in accum avg"); - Ok(result) - } - _ => { - let primitive_cnt = Primitives::Float(cnt as f64); - let result = sum_val - .as_primitive() - .map(|val| val.div(primitive_cnt)) - .map_err(|e| FnExecError::accum_error(&format!("{}", e)))?; - let result_obj: Object = result.into(); - Ok(result_obj.into()) - } - } } else { - Err(FnExecError::accum_error("unreachable")) + let cnt_primitive = Primitives::Float(cnt as f64); + let result = sum_primitive.div(cnt_primitive); + Ok(object!(result).into()) } } } @@ -276,11 +287,11 @@ impl Decode for EntryAccumulator { Ok(EntryAccumulator::ToDistinctCount(distinct_count)) } 6 => { - let sum = >::read_from(reader)?; + let sum = >::read_from(reader)?; Ok(EntryAccumulator::ToSum(sum)) } 7 => { - let sum = >::read_from(reader)?; + let sum = >::read_from(reader)?; let count = >::read_from(reader)?; Ok(EntryAccumulator::ToAvg(sum, count)) } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs index a0ec388fa91e..4b6fd96790e3 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs @@ -23,7 +23,7 @@ use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; use crate::process::entry::{CollectionEntry, EntryDataType}; use crate::process::operator::flatmap::FlatMapFuncGen; -use crate::process::operator::map::Intersection; +use crate::process::operator::map::IntersectionEntry; use crate::process::record::Record; #[derive(Debug)] @@ -68,7 +68,7 @@ impl FlatMapFunction for UnfoldOperator { EntryDataType::Intersect => { let intersection = entry .as_any_mut() - .downcast_mut::() + .downcast_mut::() .unwrap(); let mut res = Vec::with_capacity(intersection.len()); for item in intersection.drain() { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index 6c7b7f68ae7f..d152d5f21107 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -49,15 +49,15 @@ struct ExpandEdgeVOrIntersect { } #[derive(Debug, Clone, Hash, PartialEq, PartialOrd)] -pub struct Intersection { +pub struct IntersectionEntry { vertex_vec: Vec, count_vec: Vec, } -impl_as_any!(Intersection); +impl_as_any!(IntersectionEntry); -impl Intersection { - pub fn from_iter>(iter: I) -> Intersection { +impl IntersectionEntry { + pub fn from_iter>(iter: I) -> IntersectionEntry { let mut vertex_count_map = BTreeMap::new(); for vertex in iter { let cnt = vertex_count_map.entry(vertex).or_insert(0); @@ -69,7 +69,7 @@ impl Intersection { vertex_vec.push(vertex); count_vec.push(cnt); } - Intersection { vertex_vec, count_vec } + IntersectionEntry { vertex_vec, count_vec } } fn intersect>(&mut self, seeker: Iter) { @@ -124,7 +124,7 @@ impl Intersection { } } -impl Encode for Intersection { +impl Encode for IntersectionEntry { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { self.vertex_vec.write_to(writer)?; self.count_vec.write_to(writer)?; @@ -132,15 +132,15 @@ impl Encode for Intersection { } } -impl Decode for Intersection { +impl Decode for IntersectionEntry { fn read_from(reader: &mut R) -> std::io::Result { let vertex_vec = >::read_from(reader)?; let count_vec = >::read_from(reader)?; - Ok(Intersection { vertex_vec, count_vec }) + Ok(IntersectionEntry { vertex_vec, count_vec }) } } -impl Element for Intersection { +impl Element for IntersectionEntry { fn len(&self) -> usize { self.len() } @@ -164,7 +164,7 @@ impl FilterMapFunction for ExpandVertexOrIntersect { // the case of expansion and intersection let pre_intersection = pre_entry .as_any_mut() - .downcast_mut::() + .downcast_mut::() .ok_or(FnExecError::unexpected_data_error(&format!( "entry is not a intersection in ExpandOrIntersect" )))?; @@ -176,7 +176,7 @@ impl FilterMapFunction for ExpandVertexOrIntersect { } } else { // the case of expansion only - let neighbors_intersection = Intersection::from_iter(iter); + let neighbors_intersection = IntersectionEntry::from_iter(iter); if neighbors_intersection.is_empty() { Ok(None) } else { @@ -212,7 +212,7 @@ impl FilterMapFunction for ExpandEdgeVOrIntersect { // the case of expansion and intersection let pre_intersection = pre_entry .as_any_mut() - .downcast_mut::() + .downcast_mut::() .ok_or(FnExecError::unexpected_data_error(&format!( "entry is not a intersection in ExpandOrIntersect" )))?; @@ -224,7 +224,7 @@ impl FilterMapFunction for ExpandEdgeVOrIntersect { } } else { // the case of expansion only - let neighbors_intersection = Intersection::from_iter(iter); + let neighbors_intersection = IntersectionEntry::from_iter(iter); if neighbors_intersection.is_empty() { Ok(None) } else { @@ -288,7 +288,7 @@ mod tests { use graph_proxy::apis::{GraphElement, Vertex, ID}; - use super::Intersection; + use super::IntersectionEntry; fn to_vertex_iter(id_vec: Vec) -> impl Iterator { id_vec.into_iter().map(|id| id.into()) @@ -296,7 +296,7 @@ mod tests { #[test] fn intersect_test_01() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 2, 3, 4, 5]); intersection.intersect(seeker); assert_eq!( @@ -310,7 +310,7 @@ mod tests { #[test] fn intersect_test_02() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![3, 2, 1]); intersection.intersect(seeker); assert_eq!( @@ -324,7 +324,7 @@ mod tests { #[test] fn intersect_test_03() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![9, 7, 5, 3, 1]); intersection.intersect(seeker); assert_eq!( @@ -338,7 +338,7 @@ mod tests { #[test] fn intersect_test_04() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![9, 8, 7, 6]); intersection.intersect(seeker); assert_eq!( @@ -352,7 +352,7 @@ mod tests { #[test] fn intersect_test_05() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5, 1])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5, 1])); let seeker = to_vertex_iter(vec![1, 2, 3]); intersection.intersect(seeker); assert_eq!( @@ -366,7 +366,7 @@ mod tests { #[test] fn intersect_test_06() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 2, 3, 4, 5, 1]); intersection.intersect(seeker); assert_eq!( @@ -380,7 +380,7 @@ mod tests { #[test] fn intersect_test_07() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5])); let seeker = to_vertex_iter(vec![1, 2, 3]); intersection.intersect(seeker); assert_eq!( @@ -394,7 +394,7 @@ mod tests { #[test] fn intersect_test_08() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 2, 3])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5]); intersection.intersect(seeker); assert_eq!( @@ -408,7 +408,7 @@ mod tests { #[test] fn intersect_test_09() { - let mut intersection = Intersection::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3])); + let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3])); let seeker = to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5]); intersection.intersect(seeker); assert_eq!( diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/mod.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/mod.rs index 217aab4a3a63..b17464a3d453 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/mod.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/mod.rs @@ -19,7 +19,7 @@ mod path_end; mod path_start; mod project; -pub use expand_intersect::Intersection; +pub use expand_intersect::IntersectionEntry; use ir_common::error::ParsePbError; use ir_common::generated::algebra as algebra_pb; use pegasus::api::function::{FilterMapFunction, MapFunction}; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs index 341b3d05becd..c5ddbb835279 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs @@ -30,7 +30,7 @@ use prost::Message; use crate::error::FnGenResult; use crate::process::entry::{CollectionEntry, DynEntry, Entry, EntryDataType}; -use crate::process::operator::map::Intersection; +use crate::process::operator::map::IntersectionEntry; use crate::process::operator::sink::{SinkGen, Sinker}; use crate::process::record::Record; @@ -62,7 +62,7 @@ impl RecordSinkEncoder { EntryDataType::Intersect => { let intersection = e .as_any_ref() - .downcast_ref::() + .downcast_ref::() .unwrap(); let mut collection_pb = Vec::with_capacity(intersection.len()); for v in intersection.iter() { From a3158542ea420cc85f551a628b1ab79444842289 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 17 Jan 2023 10:08:06 +0800 Subject: [PATCH 6/8] [GIE/Runtime] Use IdEntry for opt; and refine the codes --- .../ir/integrated/tests/expand_test.rs | 18 +- .../executor/ir/runtime/src/process/entry.rs | 181 ++++++-------- .../src/process/operator/accum/accum.rs | 21 +- .../process/operator/flatmap/edge_expand.rs | 92 +++---- .../src/process/operator/flatmap/unfold.rs | 10 +- .../src/process/operator/map/auxilia.rs | 13 +- .../process/operator/map/expand_intersect.rs | 234 +++++------------- .../runtime/src/process/operator/shuffle.rs | 16 +- .../runtime/src/process/operator/sink/sink.rs | 38 +-- .../executor/ir/runtime/src/process/record.rs | 6 +- 10 files changed, 262 insertions(+), 367 deletions(-) diff --git a/interactive_engine/executor/ir/integrated/tests/expand_test.rs b/interactive_engine/executor/ir/integrated/tests/expand_test.rs index 13084c4e9438..ae68fa317611 100644 --- a/interactive_engine/executor/ir/integrated/tests/expand_test.rs +++ b/interactive_engine/executor/ir/integrated/tests/expand_test.rs @@ -444,7 +444,7 @@ mod test { let expected_ids = vec![2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); assert!(element .details() @@ -493,7 +493,7 @@ mod test { let expected_ids = vec![1, 4, 4, 6]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); assert!(element .details() @@ -542,7 +542,7 @@ mod test { let expected_ids = vec![1, 1, 2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); assert!(element .details() @@ -591,7 +591,7 @@ mod test { let expected_ids = vec![1, 1, 2, 4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); assert!(element .details() @@ -761,7 +761,7 @@ mod test { let mut result_collection: Vec = intersection .clone() .iter() - .map(|r| r.id() as usize) + .map(|r| *r as usize) .collect(); result_collection.sort(); result_collections.push(result_collection); @@ -839,7 +839,7 @@ mod test { let mut result_collection: Vec = intersection .clone() .iter() - .map(|r| r.id() as DefaultId) + .map(|r| *r as DefaultId) .collect(); result_collection.sort(); result_collections.push(result_collection); @@ -912,7 +912,7 @@ mod test { let expected_ids = vec![v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); } } @@ -980,7 +980,7 @@ mod test { let mut expected_ids = vec![v1, v1, v3, v3, v4, v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); } } @@ -1058,7 +1058,7 @@ mod test { let mut expected_ids = vec![v4]; let mut result_ids = vec![]; while let Some(Ok(record)) = result.next() { - if let Some(element) = record.get(None).unwrap().as_vertex() { + if let Some(element) = record.get(None) { result_ids.push(element.id() as usize); } } diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs index 80f1b270740b..6b4fca5eed22 100644 --- a/interactive_engine/executor/ir/runtime/src/process/entry.rs +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -34,47 +34,44 @@ use pegasus_common::impl_as_any; use crate::process::operator::map::IntersectionEntry; #[derive(Debug)] -pub enum EntryDataType { +pub enum EntryType { // TODO: Specified as a vertex global id for tmp; // After we separate GetVertexProperty and GetEdgeProperty in `Auxilia`, this would be an `Id`. - Vid, + VID, /// Graph Vertex - V, + VERTEX, /// Graph Edge - E, + EDGE, /// Graph Path - P, + PATH, /// Common data type of `Object`, including Primitives, String, etc. - Obj, + OBJECT, /// A specific type used in `ExtendIntersect`, for an optimized implementation of `Intersection` - Intersect, + INTERSECT, /// Type of collection consisting of entries - Collection, + COLLECTION, } -impl PartialEq for EntryDataType { +impl PartialEq for EntryType { fn eq(&self, other: &Self) -> bool { match (self, other) { - (EntryDataType::Vid, EntryDataType::Vid) - | (EntryDataType::Vid, EntryDataType::V) - | (EntryDataType::V, EntryDataType::Vid) - | (EntryDataType::V, EntryDataType::V) - | (EntryDataType::E, EntryDataType::E) - | (EntryDataType::P, EntryDataType::P) - | (EntryDataType::Obj, EntryDataType::Obj) - | (EntryDataType::Intersect, EntryDataType::Intersect) - | (EntryDataType::Collection, EntryDataType::Collection) => true, + (EntryType::VID, EntryType::VID) + | (EntryType::VID, EntryType::VERTEX) + | (EntryType::VERTEX, EntryType::VID) + | (EntryType::VERTEX, EntryType::VERTEX) + | (EntryType::EDGE, EntryType::EDGE) + | (EntryType::PATH, EntryType::PATH) + | (EntryType::OBJECT, EntryType::OBJECT) + | (EntryType::INTERSECT, EntryType::INTERSECT) + | (EntryType::COLLECTION, EntryType::COLLECTION) => true, _ => false, } } } pub trait Entry: Debug + Send + Sync + AsAny + Element { - fn get_type(&self) -> EntryDataType; - // TODO: would be `as_id()` - fn as_vid(&self) -> Option { - None - } + fn get_type(&self) -> EntryType; + fn as_vertex(&self) -> Option<&Vertex> { None } @@ -119,7 +116,7 @@ impl DynEntry { pub fn is_none(&self) -> bool { match self.get_type() { - EntryDataType::Obj => self + EntryType::OBJECT => self .as_object() .map(|obj| obj.eq(&Object::None)) .unwrap_or(false), @@ -129,14 +126,10 @@ impl DynEntry { } impl Entry for DynEntry { - fn get_type(&self) -> EntryDataType { + fn get_type(&self) -> EntryType { self.inner.get_type() } - fn as_vid(&self) -> Option { - self.inner.as_vid() - } - fn as_vertex(&self) -> Option<&Vertex> { self.inner.as_vertex() } @@ -158,27 +151,27 @@ impl Encode for DynEntry { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { let entry_type = self.get_type(); match entry_type { - EntryDataType::Vid => { + EntryType::VID => { writer.write_u8(0)?; - write_id(writer, self.as_vid().unwrap())?; + write_id(writer, self.id())?; } - EntryDataType::V => { + EntryType::VERTEX => { writer.write_u8(1)?; self.as_vertex().unwrap().write_to(writer)?; } - EntryDataType::E => { + EntryType::EDGE => { writer.write_u8(2)?; self.as_edge().unwrap().write_to(writer)?; } - EntryDataType::P => { + EntryType::PATH => { writer.write_u8(3)?; self.as_graph_path().unwrap().write_to(writer)?; } - EntryDataType::Obj => { + EntryType::OBJECT => { writer.write_u8(4)?; self.as_object().unwrap().write_to(writer)?; } - EntryDataType::Intersect => { + EntryType::INTERSECT => { writer.write_u8(5)?; self.inner .as_any_ref() @@ -186,7 +179,7 @@ impl Encode for DynEntry { .unwrap() .write_to(writer)?; } - EntryDataType::Collection => { + EntryType::COLLECTION => { writer.write_u8(6)?; self.inner .as_any_ref() @@ -253,39 +246,28 @@ impl Element for DynEntry { impl GraphElement for DynEntry { fn id(&self) -> ID { match self.get_type() { - EntryDataType::Vid | EntryDataType::V => self.as_vid().unwrap(), - EntryDataType::E => self.as_edge().unwrap().id(), + EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { + self.inner.as_graph_element().unwrap().id() + } _ => unreachable!(), } } fn label(&self) -> Option { match self.get_type() { - EntryDataType::Vid => None, - EntryDataType::V => self - .as_vertex() - .map(|v| v.label()) - .unwrap_or(None), - EntryDataType::E => self - .as_edge() - .map(|v| v.label()) - .unwrap_or(None), - _ => unreachable!(), + EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { + self.inner.as_graph_element().unwrap().label() + } + _ => None, } } fn details(&self) -> Option<&DynDetails> { match self.get_type() { - EntryDataType::Vid => None, - EntryDataType::V => self - .as_vertex() - .map(|v| v.details()) - .unwrap_or(None), - EntryDataType::E => self - .as_edge() - .map(|v| v.details()) - .unwrap_or(None), - _ => unreachable!(), + EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { + self.inner.as_graph_element().unwrap().details() + } + _ => None, } } } @@ -294,15 +276,16 @@ impl GraphElement for DynEntry { impl Hash for DynEntry { fn hash(&self, state: &mut H) { match self.get_type() { - EntryDataType::Vid | EntryDataType::V => self.as_vid().hash(state), - EntryDataType::E => self.as_edge().hash(state), - EntryDataType::P => self.as_graph_path().hash(state), - EntryDataType::Obj => self.as_object().hash(state), - EntryDataType::Intersect => self + EntryType::VID => self.id().hash(state), + EntryType::VERTEX => self.as_vertex().hash(state), + EntryType::EDGE => self.as_edge().hash(state), + EntryType::PATH => self.as_graph_path().hash(state), + EntryType::OBJECT => self.as_object().hash(state), + EntryType::INTERSECT => self .as_any_ref() .downcast_ref::() .hash(state), - EntryDataType::Collection => self + EntryType::COLLECTION => self .as_any_ref() .downcast_ref::() .hash(state), @@ -315,17 +298,18 @@ impl PartialEq for DynEntry { fn eq(&self, other: &Self) -> bool { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryDataType::Vid | EntryDataType::V => self.as_vid().eq(&other.as_vid()), - EntryDataType::E => self.as_edge().eq(&other.as_edge()), - EntryDataType::P => self.as_graph_path().eq(&other.as_graph_path()), - EntryDataType::Obj => self.as_object().eq(&other.as_object()), - EntryDataType::Intersect => self + EntryType::VID => self.id().eq(&other.id()), + EntryType::VERTEX => self.as_vertex().eq(&other.as_vertex()), + EntryType::EDGE => self.as_edge().eq(&other.as_edge()), + EntryType::PATH => self.as_graph_path().eq(&other.as_graph_path()), + EntryType::OBJECT => self.as_object().eq(&other.as_object()), + EntryType::INTERSECT => self .as_any_ref() .downcast_ref::() .eq(&other .as_any_ref() .downcast_ref::()), - EntryDataType::Collection => self + EntryType::COLLECTION => self .as_any_ref() .downcast_ref::() .eq(&other @@ -343,13 +327,14 @@ impl PartialOrd for DynEntry { fn partial_cmp(&self, other: &Self) -> Option { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryDataType::Vid | EntryDataType::V => self.as_vid().partial_cmp(&other.as_vid()), - EntryDataType::E => self.as_edge().partial_cmp(&other.as_edge()), - EntryDataType::P => self + EntryType::VID => self.id().partial_cmp(&other.id()), + EntryType::VERTEX => self.as_vertex().partial_cmp(&other.as_vertex()), + EntryType::EDGE => self.as_edge().partial_cmp(&other.as_edge()), + EntryType::PATH => self .as_graph_path() .partial_cmp(&other.as_graph_path()), - EntryDataType::Obj => self.as_object().partial_cmp(&other.as_object()), - EntryDataType::Intersect => self + EntryType::OBJECT => self.as_object().partial_cmp(&other.as_object()), + EntryType::INTERSECT => self .as_any_ref() .downcast_ref::() .partial_cmp( @@ -357,7 +342,7 @@ impl PartialOrd for DynEntry { .as_any_ref() .downcast_ref::(), ), - EntryDataType::Collection => self + EntryType::COLLECTION => self .as_any_ref() .downcast_ref::() .partial_cmp( @@ -380,15 +365,17 @@ pub struct IdEntry { id: ID, } +impl IdEntry { + pub fn new(id: ID) -> Self { + IdEntry { id } + } +} + impl_as_any!(IdEntry); impl Entry for IdEntry { - fn get_type(&self) -> EntryDataType { - EntryDataType::Vid - } - - fn as_vid(&self) -> Option { - Some(self.id) + fn get_type(&self) -> EntryType { + EntryType::VID } } @@ -430,12 +417,8 @@ impl GraphElement for IdEntry { } impl Entry for Vertex { - fn get_type(&self) -> EntryDataType { - EntryDataType::V - } - - fn as_vid(&self) -> Option { - Some(self.id()) + fn get_type(&self) -> EntryType { + EntryType::VERTEX } fn as_vertex(&self) -> Option<&Vertex> { @@ -444,8 +427,8 @@ impl Entry for Vertex { } impl Entry for Edge { - fn get_type(&self) -> EntryDataType { - EntryDataType::E + fn get_type(&self) -> EntryType { + EntryType::EDGE } fn as_edge(&self) -> Option<&Edge> { @@ -454,8 +437,8 @@ impl Entry for Edge { } impl Entry for Object { - fn get_type(&self) -> EntryDataType { - EntryDataType::Obj + fn get_type(&self) -> EntryType { + EntryType::OBJECT } fn as_object(&self) -> Option<&Object> { @@ -464,14 +447,14 @@ impl Entry for Object { } impl Entry for IntersectionEntry { - fn get_type(&self) -> EntryDataType { - EntryDataType::Intersect + fn get_type(&self) -> EntryType { + EntryType::INTERSECT } } impl Entry for GraphPath { - fn get_type(&self) -> EntryDataType { - EntryDataType::P + fn get_type(&self) -> EntryType { + EntryType::PATH } fn as_graph_path(&self) -> Option<&GraphPath> { @@ -487,8 +470,8 @@ pub struct CollectionEntry { impl_as_any!(CollectionEntry); impl Entry for CollectionEntry { - fn get_type(&self) -> EntryDataType { - EntryDataType::Collection + fn get_type(&self) -> EntryType { + EntryType::COLLECTION } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs index d320d4969b76..2b4a1bd775b4 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/accum/accum.rs @@ -25,7 +25,7 @@ use ir_common::KeyId; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use crate::error::{FnExecError, FnExecResult, FnGenError, FnGenResult}; -use crate::process::entry::{DynEntry, Entry}; +use crate::process::entry::{CollectionEntry, DynEntry, Entry}; use crate::process::operator::accum::accumulator::{ Accumulator, Count, DistinctCount, Maximum, Minimum, Sum, ToList, ToSet, }; @@ -120,11 +120,11 @@ impl Accumulator for EntryAccumulator { match self { EntryAccumulator::ToCount(count) => { let cnt = count.finalize()?; - Ok(object!(cnt).into()) + Ok(DynEntry::new(object!(cnt))) } EntryAccumulator::ToList(list) => { - let list_entry: Vec = list.finalize()?; - Ok(list_entry.into()) + let list_entry = CollectionEntry { inner: list.finalize()? }; + Ok(DynEntry::new(list_entry)) } EntryAccumulator::ToMin(min) => min .finalize()? @@ -133,18 +133,18 @@ impl Accumulator for EntryAccumulator { .finalize()? .ok_or(FnExecError::accum_error("max_entry is none")), EntryAccumulator::ToSet(set) => { - let set_entry = set.finalize()?; - Ok(set_entry.into()) + let set_entry = CollectionEntry { inner: set.finalize()? }; + Ok(DynEntry::new(set_entry)) } EntryAccumulator::ToDistinctCount(distinct_count) => { let cnt = distinct_count.finalize()?; - Ok(object!(cnt).into()) + Ok(DynEntry::new(object!(cnt))) } EntryAccumulator::ToSum(sum) => { let primitive = sum .finalize()? .ok_or(FnExecError::accum_error("sum_entry is none"))?; - Ok(object!(primitive).into()) + Ok(DynEntry::new(object!(primitive))) } EntryAccumulator::ToAvg(sum, count) => { let sum_primitive = sum @@ -152,14 +152,13 @@ impl Accumulator for EntryAccumulator { .ok_or(FnExecError::accum_error("sum_entry is none"))?; let cnt = count.finalize()?; // TODO: confirm if it should be Object::None, or throw error; - let result = (Object::None).into(); if cnt == 0 { warn!("cnt value is 0 in accum avg"); - Ok(result) + Ok(DynEntry::new(Object::None)) } else { let cnt_primitive = Primitives::Float(cnt as f64); let result = sum_primitive.div(cnt_primitive); - Ok(object!(result).into()) + Ok(DynEntry::new(object!(result))) } } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs index a1b5ad725425..9167aa8fe2bc 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs @@ -24,7 +24,7 @@ use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenError, FnGenResult}; -use crate::process::entry::Entry; +use crate::process::entry::{Entry, EntryType}; use crate::process::operator::flatmap::FlatMapFuncGen; use crate::process::record::{Record, RecordExpandIter, RecordPathExpandIter}; @@ -40,52 +40,58 @@ impl FlatMapFunction for EdgeExpandOperator< fn exec(&self, mut input: Record) -> FnResult { if let Some(entry) = input.get(self.start_v_tag) { - if let Some(id) = entry.as_vid() { - let iter = self.stmt.exec(id)?; - match self.expand_opt { - // the case of expand edge, and get end vertex; - ExpandOpt::Vertex => { - let neighbors_iter = iter.map(|e| { - if let Some(e) = e.as_edge() { - Vertex::new( - e.get_other_id(), - e.get_other_label().cloned(), - DynDetails::default(), - ) - } else { - unreachable!() - } - }); - Ok(Box::new(RecordExpandIter::new( - input, - self.alias.as_ref(), - Box::new(neighbors_iter), - ))) - } - // the case of expand neighbors, including edges/vertices - ExpandOpt::Edge => { - Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) - } - // the case of get degree. TODO: this case should be a `Map` - ExpandOpt::Degree => { - let degree = iter.count(); - input.append(object!(degree), self.alias); - Ok(Box::new(vec![input].into_iter())) + match entry.get_type() { + EntryType::VID | EntryType::VERTEX => { + let id = entry.id(); + let iter = self.stmt.exec(id)?; + match self.expand_opt { + // the case of expand edge, and get end vertex; + ExpandOpt::Vertex => { + let neighbors_iter = iter.map(|e| { + if let Some(e) = e.as_edge() { + Vertex::new( + e.get_other_id(), + e.get_other_label().cloned(), + DynDetails::default(), + ) + } else { + unreachable!() + } + }); + Ok(Box::new(RecordExpandIter::new( + input, + self.alias.as_ref(), + Box::new(neighbors_iter), + ))) + } + // the case of expand neighbors, including edges/vertices + ExpandOpt::Edge => { + Ok(Box::new(RecordExpandIter::new(input, self.alias.as_ref(), iter))) + } + // the case of get degree. TODO: this case should be a `Map` + ExpandOpt::Degree => { + let degree = iter.count(); + input.append(object!(degree), self.alias); + Ok(Box::new(vec![input].into_iter())) + } } } - } else if let Some(graph_path) = entry.as_graph_path() { - let path_end = graph_path - .get_path_end() - .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; - let id = path_end.id(); - let iter = self.stmt.exec(id)?; - let curr_path = graph_path.clone(); - Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) - } else { - Err(FnExecError::unexpected_data_error(&format!( + EntryType::PATH => { + let graph_path = entry + .as_graph_path() + .ok_or(FnExecError::Unreachable)?; + let path_end = graph_path + .get_path_end() + .ok_or(FnExecError::unexpected_data_error("Get path_end failed in path expand"))?; + let id = path_end.id(); + let iter = self.stmt.exec(id)?; + let curr_path = graph_path.clone(); + Ok(Box::new(RecordPathExpandIter::new(input, curr_path, iter))) + } + _ => Err(FnExecError::unexpected_data_error(&format!( "Cannot Expand from current entry {:?}", entry - )))? + )))?, } } else { Ok(Box::new(vec![].into_iter())) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs index 4b6fd96790e3..0da1f4eba3de 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs @@ -21,7 +21,7 @@ use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; -use crate::process::entry::{CollectionEntry, EntryDataType}; +use crate::process::entry::{CollectionEntry, EntryType, IdEntry}; use crate::process::operator::flatmap::FlatMapFuncGen; use crate::process::operator::map::IntersectionEntry; use crate::process::record::Record; @@ -52,7 +52,7 @@ impl FlatMapFunction for UnfoldOperator { input.take(None); if let Some(entry) = entry.get_mut() { match entry.get_type() { - EntryDataType::Collection => { + EntryType::COLLECTION => { let collection = entry .as_any_mut() .downcast_mut::() @@ -65,7 +65,7 @@ impl FlatMapFunction for UnfoldOperator { } Ok(Box::new(res.into_iter())) } - EntryDataType::Intersect => { + EntryType::INTERSECT => { let intersection = entry .as_any_mut() .downcast_mut::() @@ -73,12 +73,12 @@ impl FlatMapFunction for UnfoldOperator { let mut res = Vec::with_capacity(intersection.len()); for item in intersection.drain() { let mut new_entry = input.clone(); - new_entry.append(item, self.alias); + new_entry.append(IdEntry::new(item), self.alias); res.push(new_entry); } Ok(Box::new(res.into_iter())) } - EntryDataType::P => Err(FnExecError::unsupported_error(&format!( + EntryType::PATH => Err(FnExecError::unsupported_error(&format!( "unfold path entry {:?} in UnfoldOperator", entry )))?, diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs index 179e65867daf..acff7787d1ee 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs @@ -21,7 +21,7 @@ use ir_common::KeyId; use pegasus::api::function::{FilterMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; -use crate::process::entry::{DynEntry, Entry, EntryDataType}; +use crate::process::entry::{DynEntry, Entry, EntryType}; use crate::process::operator::map::FilterMapFuncGen; use crate::process::record::Record; @@ -46,8 +46,8 @@ impl FilterMapFunction for AuxiliaOperator { // If queryable, then turn into graph element and do the query let graph = get_graph().ok_or(FnExecError::NullGraphError)?; let new_entry: Option = match entry.get_type() { - EntryDataType::Vid | EntryDataType::V => { - let id = entry.as_vid().unwrap(); + EntryType::VID | EntryType::VERTEX => { + let id = entry.id(); let mut result_iter = graph.get_vertex(&[id], &self.query_params)?; result_iter.next().map(|mut vertex| { // TODO:confirm the update case, and avoid it if possible. @@ -67,11 +67,8 @@ impl FilterMapFunction for AuxiliaOperator { DynEntry::new(vertex) }) } - EntryDataType::E => { - let id = entry - .as_edge() - .ok_or(FnExecError::Unreachable)? - .id(); + EntryType::EDGE => { + let id = entry.id(); let mut result_iter = graph.get_edge(&[id], &self.query_params)?; result_iter.next().map(|mut edge| { if let Some(details) = entry diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index d152d5f21107..4d5ae169b2f7 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -19,7 +19,7 @@ use std::convert::TryInto; use dyn_type::BorrowObject; use graph_proxy::apis::graph::element::GraphElement; -use graph_proxy::apis::{Direction, DynDetails, Edge, Element, QueryParams, Statement, Vertex, ID}; +use graph_proxy::apis::{Direction, Element, QueryParams, Statement, ID}; use ir_common::error::ParsePbError; use ir_common::generated::algebra as algebra_pb; use ir_common::KeyId; @@ -29,35 +29,29 @@ use pegasus_common::downcast::*; use pegasus_common::impl_as_any; use crate::error::{FnExecError, FnGenError, FnGenResult}; -use crate::process::entry::{DynEntry, Entry}; +use crate::process::entry::{DynEntry, Entry, EntryType}; use crate::process::operator::map::FilterMapFuncGen; use crate::process::record::Record; /// An ExpandOrIntersect operator to expand neighbors /// and intersect with the ones of the same tag found previously (if exists). /// Notice that edge_or_end_v_tag (the alias of expanded neighbors) must be specified. -struct ExpandVertexOrIntersect { +struct ExpandOrIntersect { start_v_tag: Option, edge_or_end_v_tag: KeyId, - stmt: Box>, -} - -struct ExpandEdgeVOrIntersect { - start_v_tag: Option, - edge_or_end_v_tag: KeyId, - stmt: Box>, + stmt: Box>, } #[derive(Debug, Clone, Hash, PartialEq, PartialOrd)] pub struct IntersectionEntry { - vertex_vec: Vec, + vertex_vec: Vec, count_vec: Vec, } impl_as_any!(IntersectionEntry); impl IntersectionEntry { - pub fn from_iter>(iter: I) -> IntersectionEntry { + pub fn from_iter>(iter: I) -> IntersectionEntry { let mut vertex_count_map = BTreeMap::new(); for vertex in iter { let cnt = vertex_count_map.entry(vertex).or_insert(0); @@ -72,14 +66,13 @@ impl IntersectionEntry { IntersectionEntry { vertex_vec, count_vec } } - fn intersect>(&mut self, seeker: Iter) { + fn intersect>(&mut self, seeker: Iter) { let len = self.vertex_vec.len(); let mut s = vec![0; len]; - for item in seeker { - let vid = item.id(); + for vid in seeker { if let Ok(idx) = self .vertex_vec - .binary_search_by(|e| e.id().cmp(&vid)) + .binary_search_by(|e| e.cmp(&vid)) { s[idx] += 1; } @@ -97,11 +90,11 @@ impl IntersectionEntry { self.count_vec.drain(idx..); } - pub fn is_empty(&self) -> bool { + fn is_empty(&self) -> bool { self.vertex_vec.is_empty() } - pub fn len(&self) -> usize { + fn len(&self) -> usize { let mut len = 0; for count in self.count_vec.iter() { len += *count; @@ -109,14 +102,14 @@ impl IntersectionEntry { len as usize } - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.vertex_vec .iter() .zip(&self.count_vec) .flat_map(move |(vertex, count)| std::iter::repeat(vertex).take(*count as usize)) } - pub fn drain(&mut self) -> impl Iterator + '_ { + pub fn drain(&mut self) -> impl Iterator + '_ { self.vertex_vec .drain(..) .zip(&self.count_vec) @@ -134,7 +127,7 @@ impl Encode for IntersectionEntry { impl Decode for IntersectionEntry { fn read_from(reader: &mut R) -> std::io::Result { - let vertex_vec = >::read_from(reader)?; + let vertex_vec = >::read_from(reader)?; let count_vec = >::read_from(reader)?; Ok(IntersectionEntry { vertex_vec, count_vec }) } @@ -150,7 +143,7 @@ impl Element for IntersectionEntry { } } -impl FilterMapFunction for ExpandVertexOrIntersect { +impl FilterMapFunction for ExpandOrIntersect { fn exec(&self, mut input: Record) -> FnResult> { let entry = input .get(self.start_v_tag) @@ -158,87 +151,50 @@ impl FilterMapFunction for ExpandVertexOrIntersect { "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", self.start_v_tag, input )))?; - if let Some(id) = entry.as_vid() { - let iter = self.stmt.exec(id)?; - if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { - // the case of expansion and intersection - let pre_intersection = pre_entry - .as_any_mut() - .downcast_mut::() - .ok_or(FnExecError::unexpected_data_error(&format!( - "entry is not a intersection in ExpandOrIntersect" - )))?; - pre_intersection.intersect(iter); - if pre_intersection.is_empty() { - Ok(None) - } else { - Ok(Some(input)) - } - } else { - // the case of expansion only - let neighbors_intersection = IntersectionEntry::from_iter(iter); - if neighbors_intersection.is_empty() { - Ok(None) + match entry.get_type() { + EntryType::VID | EntryType::VERTEX => { + let id = entry.id(); + let iter = self.stmt.exec(id)?.map(|e| { + if let Some(vertex) = e.as_vertex() { + vertex.id() as ID + } else if let Some(edge) = e.as_edge() { + edge.get_other_id() as ID + } else { + unreachable!() + } + }); + if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { + // the case of expansion and intersection + let pre_intersection = pre_entry + .as_any_mut() + .downcast_mut::() + .ok_or(FnExecError::unexpected_data_error(&format!( + "entry is not a intersection in ExpandOrIntersect" + )))?; + pre_intersection.intersect(iter); + if pre_intersection.is_empty() { + Ok(None) + } else { + Ok(Some(input)) + } } else { - // append columns without changing head - let columns = input.get_columns_mut(); - columns.insert(self.edge_or_end_v_tag as usize, DynEntry::new(neighbors_intersection)); - Ok(Some(input)) + // the case of expansion only + let neighbors_intersection = IntersectionEntry::from_iter(iter); + if neighbors_intersection.is_empty() { + Ok(None) + } else { + // append columns without changing head + let columns = input.get_columns_mut(); + columns + .insert(self.edge_or_end_v_tag as usize, DynEntry::new(neighbors_intersection)); + Ok(Some(input)) + } } } - } else { - Err(FnExecError::unsupported_error(&format!( - "expand or intersect entry {:?} of tag {:?} failed in ExpandOrIntersect", - entry, self.edge_or_end_v_tag - )))? - } - } -} - -impl FilterMapFunction for ExpandEdgeVOrIntersect { - fn exec(&self, mut input: Record) -> FnResult> { - let entry = input - .get(self.start_v_tag) - .ok_or(FnExecError::get_tag_error(&format!( - "get start_v_tag {:?} from record in `ExpandOrIntersect` operator, the record is {:?}", - self.start_v_tag, input - )))?; - if let Some(id) = entry.as_vid() { - let iter = self - .stmt - .exec(id)? - .map(|e| Vertex::new(e.get_other_id(), None, DynDetails::default())); - if let Some(pre_entry) = input.get_column_mut(&self.edge_or_end_v_tag) { - // the case of expansion and intersection - let pre_intersection = pre_entry - .as_any_mut() - .downcast_mut::() - .ok_or(FnExecError::unexpected_data_error(&format!( - "entry is not a intersection in ExpandOrIntersect" - )))?; - pre_intersection.intersect(iter); - if pre_intersection.is_empty() { - Ok(None) - } else { - Ok(Some(input)) - } - } else { - // the case of expansion only - let neighbors_intersection = IntersectionEntry::from_iter(iter); - if neighbors_intersection.is_empty() { - Ok(None) - } else { - // append columns without changing head - let columns = input.get_columns_mut(); - columns.insert(self.edge_or_end_v_tag as usize, DynEntry::new(neighbors_intersection)); - Ok(Some(input)) - } - } - } else { - Err(FnExecError::unsupported_error(&format!( + _ => Err(FnExecError::unsupported_error(&format!( "expand or intersect entry {:?} of tag {:?} failed in ExpandOrIntersect", entry, self.edge_or_end_v_tag - )))? + )))?, } } } @@ -271,12 +227,12 @@ impl FilterMapFuncGen for algebra_pb::EdgeExpand { // Expand vertices with filters on edges. // This can be regarded as a combination of EdgeExpand (with expand_opt as Edge) + GetV let stmt = graph.prepare_explore_edge(direction, &query_params)?; - let edge_expand_operator = ExpandEdgeVOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; + let edge_expand_operator = ExpandOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; Ok(Box::new(edge_expand_operator)) } else { // Expand vertices without any filters let stmt = graph.prepare_explore_vertex(direction, &query_params)?; - let edge_expand_operator = ExpandVertexOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; + let edge_expand_operator = ExpandOrIntersect { start_v_tag, edge_or_end_v_tag, stmt }; Ok(Box::new(edge_expand_operator)) } } @@ -286,12 +242,12 @@ impl FilterMapFuncGen for algebra_pb::EdgeExpand { #[cfg(test)] mod tests { - use graph_proxy::apis::{GraphElement, Vertex, ID}; + use graph_proxy::apis::ID; use super::IntersectionEntry; - fn to_vertex_iter(id_vec: Vec) -> impl Iterator { - id_vec.into_iter().map(|id| id.into()) + fn to_vertex_iter(id_vec: Vec) -> impl Iterator { + id_vec.into_iter() } #[test] @@ -299,13 +255,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 2, 3, 4, 5]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 2, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 2, 3]) } #[test] @@ -313,13 +263,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![3, 2, 1]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 2, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 2, 3]) } #[test] @@ -327,13 +271,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![9, 7, 5, 3, 1]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 3, 5] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 3, 5]) } #[test] @@ -341,13 +279,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5])); let seeker = to_vertex_iter(vec![9, 8, 7, 6]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - Vec::::new() - ) + assert_eq!(intersection.drain().collect::>(), Vec::::new()) } #[test] @@ -355,13 +287,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3, 4, 5, 1])); let seeker = to_vertex_iter(vec![1, 2, 3]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 1, 2, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 1, 2, 3]) } #[test] @@ -369,13 +295,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 2, 3, 4, 5, 1]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 1, 2, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 1, 2, 3]) } #[test] @@ -383,13 +303,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5])); let seeker = to_vertex_iter(vec![1, 2, 3]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 1, 2, 2, 3, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 1, 2, 2, 3, 3]) } #[test] @@ -397,13 +311,7 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 2, 3])); let seeker = to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 1, 2, 2, 3, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 1, 2, 2, 3, 3]) } #[test] @@ -411,12 +319,6 @@ mod tests { let mut intersection = IntersectionEntry::from_iter(to_vertex_iter(vec![1, 1, 2, 2, 3, 3])); let seeker = to_vertex_iter(vec![1, 1, 2, 2, 3, 3, 4, 5]); intersection.intersect(seeker); - assert_eq!( - intersection - .iter() - .map(|vertex| vertex.id()) - .collect::>(), - vec![1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3] - ) + assert_eq!(intersection.drain().collect::>(), vec![1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3]) } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs index b1c6af44dd8e..935a9381c599 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs @@ -23,7 +23,7 @@ use ir_common::KeyId; use pegasus::api::function::{FnResult, RouteFunction}; use crate::error::FnExecError; -use crate::process::entry::{Entry, EntryDataType}; +use crate::process::entry::{Entry, EntryType}; use crate::process::record::Record; pub struct RecordRouter { @@ -51,24 +51,22 @@ impl RouteFunction for RecordRouter { fn route(&self, t: &Record) -> FnResult { if let Some(entry) = t.get(self.shuffle_key.clone()) { match entry.get_type() { - EntryDataType::Vid | EntryDataType::V => { - let id = entry - .as_vid() - .ok_or(FnExecError::unexpected_data_error("get id failed in shuffle"))?; + EntryType::VID | EntryType::VERTEX => { + let id = entry.id(); Ok(self.p.get_partition(&id, self.num_workers)?) } - EntryDataType::E => { + EntryType::EDGE => { let e = entry .as_edge() - .ok_or(FnExecError::unexpected_data_error("get edge failed in shuffle"))?; + .ok_or(FnExecError::Unreachable)?; Ok(self .p .get_partition(&e.src_id, self.num_workers)?) } - EntryDataType::P => { + EntryType::PATH => { let p = entry .as_graph_path() - .ok_or(FnExecError::unexpected_data_error("get path failed in shuffle"))?; + .ok_or(FnExecError::Unreachable)?; let path_end = p .get_path_end() .ok_or(FnExecError::unexpected_data_error("get path_end failed in shuffle"))?; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs index c5ddbb835279..a0cae6fdbe71 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs @@ -18,7 +18,7 @@ use std::collections::HashMap; use std::convert::TryInto; use dyn_type::Object; -use graph_proxy::apis::{Edge, Element, GraphElement, GraphPath, Vertex}; +use graph_proxy::apis::{Edge, Element, GraphElement, GraphPath, Vertex, ID}; use ir_common::generated::algebra as algebra_pb; use ir_common::generated::algebra::sink_default::MetaType; use ir_common::generated::common as common_pb; @@ -29,7 +29,7 @@ use pegasus_common::downcast::AsAny; use prost::Message; use crate::error::FnGenResult; -use crate::process::entry::{CollectionEntry, DynEntry, Entry, EntryDataType}; +use crate::process::entry::{CollectionEntry, DynEntry, Entry, EntryType}; use crate::process::operator::map::IntersectionEntry; use crate::process::operator::sink::{SinkGen, Sinker}; use crate::process::record::Record; @@ -45,7 +45,7 @@ pub struct RecordSinkEncoder { impl RecordSinkEncoder { fn entry_to_pb(&self, e: &DynEntry) -> result_pb::Entry { let inner = match e.get_type() { - EntryDataType::Collection => { + EntryType::COLLECTION => { let collection = e .as_any_ref() .downcast_ref::() @@ -59,14 +59,14 @@ impl RecordSinkEncoder { collection: collection_pb, })) } - EntryDataType::Intersect => { + EntryType::INTERSECT => { let intersection = e .as_any_ref() .downcast_ref::() .unwrap(); let mut collection_pb = Vec::with_capacity(intersection.len()); for v in intersection.iter() { - let vertex_pb = self.vertex_to_pb(v); + let vertex_pb = self.vid_to_pb(v); let element_pb = result_pb::Element { inner: Some(result_pb::element::Inner::Vertex(vertex_pb)) }; collection_pb.push(element_pb); @@ -85,31 +85,32 @@ impl RecordSinkEncoder { fn element_to_pb(&self, e: &DynEntry) -> result_pb::Element { let inner = match e.get_type() { - EntryDataType::V => { + EntryType::VID => { + let vertex_pb = self.vid_to_pb(&e.id()); + Some(result_pb::element::Inner::Vertex(vertex_pb)) + } + EntryType::VERTEX => { let vertex_pb = self.vertex_to_pb(e.as_vertex().unwrap()); Some(result_pb::element::Inner::Vertex(vertex_pb)) } - EntryDataType::E => { + EntryType::EDGE => { let edge_pb = self.edge_to_pb(e.as_edge().unwrap()); Some(result_pb::element::Inner::Edge(edge_pb)) } - EntryDataType::P => { + EntryType::PATH => { let path_pb = self.path_to_pb(e.as_graph_path().unwrap()); Some(result_pb::element::Inner::GraphPath(path_pb)) } - EntryDataType::Obj => { + EntryType::OBJECT => { let obj_pb = self.object_to_pb(e.as_object().unwrap().clone()); Some(result_pb::element::Inner::Object(obj_pb)) } - EntryDataType::Collection => { + EntryType::COLLECTION => { unreachable!() } - EntryDataType::Intersect => { + EntryType::INTERSECT => { unreachable!() } - _ => { - todo!() - } }; result_pb::Element { inner } } @@ -157,6 +158,15 @@ impl RecordSinkEncoder { NameOrId::Str(meta_id.to_string()) } + fn vid_to_pb(&self, vid: &ID) -> result_pb::Vertex { + result_pb::Vertex { + id: *vid as i64, + label: None, + // TODO: return detached vertex without property for now + properties: vec![], + } + } + fn vertex_to_pb(&self, v: &Vertex) -> result_pb::Vertex { result_pb::Vertex { id: v.id() as i64, diff --git a/interactive_engine/executor/ir/runtime/src/process/record.rs b/interactive_engine/executor/ir/runtime/src/process/record.rs index b8f45f73be73..48106a79177f 100644 --- a/interactive_engine/executor/ir/runtime/src/process/record.rs +++ b/interactive_engine/executor/ir/runtime/src/process/record.rs @@ -23,7 +23,7 @@ use pegasus::api::function::DynIter; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; use vec_map::VecMap; -use crate::process::entry::{DynEntry, Entry, EntryDataType}; +use crate::process::entry::{DynEntry, Entry, EntryType}; #[derive(Debug, Clone, Default)] pub struct Record { @@ -137,8 +137,8 @@ impl Context for Record { .map(|entry| { let entry_type = entry.get_type(); match entry_type { - EntryDataType::Collection => None, - EntryDataType::Intersect => None, + EntryType::COLLECTION => None, + EntryType::INTERSECT => None, _ => Some(entry), } }) From c841c4938fad8c3320093a2fe1c34bfb6f3aedd4 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Tue, 17 Jan 2023 15:37:07 +0800 Subject: [PATCH 7/8] [GIE/Runtime] Remove `IdEntry` and `VID` type, and refine the codes --- .../executor/ir/runtime/src/process/entry.rs | 179 ++++-------------- .../process/operator/flatmap/edge_expand.rs | 4 +- .../src/process/operator/flatmap/unfold.rs | 12 +- .../src/process/operator/map/auxilia.rs | 4 +- .../process/operator/map/expand_intersect.rs | 2 +- .../runtime/src/process/operator/shuffle.rs | 6 +- .../runtime/src/process/operator/sink/sink.rs | 20 +- .../executor/ir/runtime/src/process/record.rs | 4 +- 8 files changed, 66 insertions(+), 165 deletions(-) diff --git a/interactive_engine/executor/ir/runtime/src/process/entry.rs b/interactive_engine/executor/ir/runtime/src/process/entry.rs index 6b4fca5eed22..eb13bca881a9 100644 --- a/interactive_engine/executor/ir/runtime/src/process/entry.rs +++ b/interactive_engine/executor/ir/runtime/src/process/entry.rs @@ -22,9 +22,7 @@ use std::hash::{Hash, Hasher}; use std::sync::Arc; use dyn_type::{BorrowObject, Object}; -use graph_proxy::apis::{ - read_id, write_id, DynDetails, Edge, Element, GraphElement, GraphPath, Vertex, ID, -}; +use graph_proxy::apis::{DynDetails, Edge, Element, GraphElement, GraphPath, Vertex, ID}; use ir_common::error::ParsePbError; use ir_common::generated::results as result_pb; use pegasus::codec::{Decode, Encode, ReadExt, WriteExt}; @@ -33,40 +31,20 @@ use pegasus_common::impl_as_any; use crate::process::operator::map::IntersectionEntry; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum EntryType { - // TODO: Specified as a vertex global id for tmp; - // After we separate GetVertexProperty and GetEdgeProperty in `Auxilia`, this would be an `Id`. - VID, /// Graph Vertex - VERTEX, + Vertex, /// Graph Edge - EDGE, + Edge, /// Graph Path - PATH, + Path, /// Common data type of `Object`, including Primitives, String, etc. - OBJECT, + Object, /// A specific type used in `ExtendIntersect`, for an optimized implementation of `Intersection` - INTERSECT, + Intersection, /// Type of collection consisting of entries - COLLECTION, -} - -impl PartialEq for EntryType { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (EntryType::VID, EntryType::VID) - | (EntryType::VID, EntryType::VERTEX) - | (EntryType::VERTEX, EntryType::VID) - | (EntryType::VERTEX, EntryType::VERTEX) - | (EntryType::EDGE, EntryType::EDGE) - | (EntryType::PATH, EntryType::PATH) - | (EntryType::OBJECT, EntryType::OBJECT) - | (EntryType::INTERSECT, EntryType::INTERSECT) - | (EntryType::COLLECTION, EntryType::COLLECTION) => true, - _ => false, - } - } + Collection, } pub trait Entry: Debug + Send + Sync + AsAny + Element { @@ -116,7 +94,7 @@ impl DynEntry { pub fn is_none(&self) -> bool { match self.get_type() { - EntryType::OBJECT => self + EntryType::Object => self .as_object() .map(|obj| obj.eq(&Object::None)) .unwrap_or(false), @@ -151,27 +129,23 @@ impl Encode for DynEntry { fn write_to(&self, writer: &mut W) -> std::io::Result<()> { let entry_type = self.get_type(); match entry_type { - EntryType::VID => { - writer.write_u8(0)?; - write_id(writer, self.id())?; - } - EntryType::VERTEX => { + EntryType::Vertex => { writer.write_u8(1)?; self.as_vertex().unwrap().write_to(writer)?; } - EntryType::EDGE => { + EntryType::Edge => { writer.write_u8(2)?; self.as_edge().unwrap().write_to(writer)?; } - EntryType::PATH => { + EntryType::Path => { writer.write_u8(3)?; self.as_graph_path().unwrap().write_to(writer)?; } - EntryType::OBJECT => { + EntryType::Object => { writer.write_u8(4)?; self.as_object().unwrap().write_to(writer)?; } - EntryType::INTERSECT => { + EntryType::Intersection => { writer.write_u8(5)?; self.inner .as_any_ref() @@ -179,7 +153,7 @@ impl Encode for DynEntry { .unwrap() .write_to(writer)?; } - EntryType::COLLECTION => { + EntryType::Collection => { writer.write_u8(6)?; self.inner .as_any_ref() @@ -196,10 +170,6 @@ impl Decode for DynEntry { fn read_from(reader: &mut R) -> std::io::Result { let entry_type = reader.read_u8()?; match entry_type { - 0 => { - let id = read_id(reader)?; - Ok(DynEntry::new(IdEntry { id })) - } 1 => { let vertex = Vertex::read_from(reader)?; Ok(DynEntry::new(vertex)) @@ -246,27 +216,21 @@ impl Element for DynEntry { impl GraphElement for DynEntry { fn id(&self) -> ID { match self.get_type() { - EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { - self.inner.as_graph_element().unwrap().id() - } + EntryType::Vertex | EntryType::Edge => self.inner.as_graph_element().unwrap().id(), _ => unreachable!(), } } fn label(&self) -> Option { match self.get_type() { - EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { - self.inner.as_graph_element().unwrap().label() - } + EntryType::Vertex | EntryType::Edge => self.inner.as_graph_element().unwrap().label(), _ => None, } } fn details(&self) -> Option<&DynDetails> { match self.get_type() { - EntryType::VID | EntryType::VERTEX | EntryType::EDGE => { - self.inner.as_graph_element().unwrap().details() - } + EntryType::Vertex | EntryType::Edge => self.inner.as_graph_element().unwrap().details(), _ => None, } } @@ -276,16 +240,15 @@ impl GraphElement for DynEntry { impl Hash for DynEntry { fn hash(&self, state: &mut H) { match self.get_type() { - EntryType::VID => self.id().hash(state), - EntryType::VERTEX => self.as_vertex().hash(state), - EntryType::EDGE => self.as_edge().hash(state), - EntryType::PATH => self.as_graph_path().hash(state), - EntryType::OBJECT => self.as_object().hash(state), - EntryType::INTERSECT => self + EntryType::Vertex => self.as_vertex().hash(state), + EntryType::Edge => self.as_edge().hash(state), + EntryType::Path => self.as_graph_path().hash(state), + EntryType::Object => self.as_object().hash(state), + EntryType::Intersection => self .as_any_ref() .downcast_ref::() .hash(state), - EntryType::COLLECTION => self + EntryType::Collection => self .as_any_ref() .downcast_ref::() .hash(state), @@ -298,18 +261,17 @@ impl PartialEq for DynEntry { fn eq(&self, other: &Self) -> bool { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryType::VID => self.id().eq(&other.id()), - EntryType::VERTEX => self.as_vertex().eq(&other.as_vertex()), - EntryType::EDGE => self.as_edge().eq(&other.as_edge()), - EntryType::PATH => self.as_graph_path().eq(&other.as_graph_path()), - EntryType::OBJECT => self.as_object().eq(&other.as_object()), - EntryType::INTERSECT => self + EntryType::Vertex => self.as_vertex().eq(&other.as_vertex()), + EntryType::Edge => self.as_edge().eq(&other.as_edge()), + EntryType::Path => self.as_graph_path().eq(&other.as_graph_path()), + EntryType::Object => self.as_object().eq(&other.as_object()), + EntryType::Intersection => self .as_any_ref() .downcast_ref::() .eq(&other .as_any_ref() .downcast_ref::()), - EntryType::COLLECTION => self + EntryType::Collection => self .as_any_ref() .downcast_ref::() .eq(&other @@ -327,14 +289,13 @@ impl PartialOrd for DynEntry { fn partial_cmp(&self, other: &Self) -> Option { if (self.get_type()).eq(&other.get_type()) { match self.get_type() { - EntryType::VID => self.id().partial_cmp(&other.id()), - EntryType::VERTEX => self.as_vertex().partial_cmp(&other.as_vertex()), - EntryType::EDGE => self.as_edge().partial_cmp(&other.as_edge()), - EntryType::PATH => self + EntryType::Vertex => self.as_vertex().partial_cmp(&other.as_vertex()), + EntryType::Edge => self.as_edge().partial_cmp(&other.as_edge()), + EntryType::Path => self .as_graph_path() .partial_cmp(&other.as_graph_path()), - EntryType::OBJECT => self.as_object().partial_cmp(&other.as_object()), - EntryType::INTERSECT => self + EntryType::Object => self.as_object().partial_cmp(&other.as_object()), + EntryType::Intersection => self .as_any_ref() .downcast_ref::() .partial_cmp( @@ -342,7 +303,7 @@ impl PartialOrd for DynEntry { .as_any_ref() .downcast_ref::(), ), - EntryType::COLLECTION => self + EntryType::Collection => self .as_any_ref() .downcast_ref::() .partial_cmp( @@ -360,65 +321,9 @@ impl PartialOrd for DynEntry { // demanded when need to group (ToSet) the entry; impl Eq for DynEntry {} -#[derive(Debug, Clone, Default)] -pub struct IdEntry { - id: ID, -} - -impl IdEntry { - pub fn new(id: ID) -> Self { - IdEntry { id } - } -} - -impl_as_any!(IdEntry); - -impl Entry for IdEntry { - fn get_type(&self) -> EntryType { - EntryType::VID - } -} - -impl Encode for IdEntry { - fn write_to(&self, writer: &mut W) -> std::io::Result<()> { - write_id(writer, self.id) - } -} - -impl Decode for IdEntry { - fn read_from(reader: &mut R) -> std::io::Result { - let id = read_id(reader)?; - Ok(IdEntry { id }) - } -} - -impl Element for IdEntry { - fn as_graph_element(&self) -> Option<&dyn GraphElement> { - Some(self) - } - - fn len(&self) -> usize { - 1 - } - - fn as_borrow_object(&self) -> BorrowObject { - self.id.into() - } -} - -impl GraphElement for IdEntry { - fn id(&self) -> ID { - self.id - } - - fn label(&self) -> Option { - None - } -} - impl Entry for Vertex { fn get_type(&self) -> EntryType { - EntryType::VERTEX + EntryType::Vertex } fn as_vertex(&self) -> Option<&Vertex> { @@ -428,7 +333,7 @@ impl Entry for Vertex { impl Entry for Edge { fn get_type(&self) -> EntryType { - EntryType::EDGE + EntryType::Edge } fn as_edge(&self) -> Option<&Edge> { @@ -438,7 +343,7 @@ impl Entry for Edge { impl Entry for Object { fn get_type(&self) -> EntryType { - EntryType::OBJECT + EntryType::Object } fn as_object(&self) -> Option<&Object> { @@ -448,13 +353,13 @@ impl Entry for Object { impl Entry for IntersectionEntry { fn get_type(&self) -> EntryType { - EntryType::INTERSECT + EntryType::Intersection } } impl Entry for GraphPath { fn get_type(&self) -> EntryType { - EntryType::PATH + EntryType::Path } fn as_graph_path(&self) -> Option<&GraphPath> { @@ -471,7 +376,7 @@ impl_as_any!(CollectionEntry); impl Entry for CollectionEntry { fn get_type(&self) -> EntryType { - EntryType::COLLECTION + EntryType::Collection } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs index 9167aa8fe2bc..1bb40ed80bb8 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/edge_expand.rs @@ -41,7 +41,7 @@ impl FlatMapFunction for EdgeExpandOperator< fn exec(&self, mut input: Record) -> FnResult { if let Some(entry) = input.get(self.start_v_tag) { match entry.get_type() { - EntryType::VID | EntryType::VERTEX => { + EntryType::Vertex => { let id = entry.id(); let iter = self.stmt.exec(id)?; match self.expand_opt { @@ -76,7 +76,7 @@ impl FlatMapFunction for EdgeExpandOperator< } } } - EntryType::PATH => { + EntryType::Path => { let graph_path = entry .as_graph_path() .ok_or(FnExecError::Unreachable)?; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs index 0da1f4eba3de..9f70d8623894 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/flatmap/unfold.rs @@ -15,13 +15,13 @@ use std::convert::TryInto; -use graph_proxy::apis::Element; +use graph_proxy::apis::{DynDetails, Element, Vertex}; use ir_common::generated::algebra as algebra_pb; use ir_common::KeyId; use pegasus::api::function::{DynIter, FlatMapFunction, FnResult}; use crate::error::{FnExecError, FnGenResult}; -use crate::process::entry::{CollectionEntry, EntryType, IdEntry}; +use crate::process::entry::{CollectionEntry, EntryType}; use crate::process::operator::flatmap::FlatMapFuncGen; use crate::process::operator::map::IntersectionEntry; use crate::process::record::Record; @@ -52,7 +52,7 @@ impl FlatMapFunction for UnfoldOperator { input.take(None); if let Some(entry) = entry.get_mut() { match entry.get_type() { - EntryType::COLLECTION => { + EntryType::Collection => { let collection = entry .as_any_mut() .downcast_mut::() @@ -65,7 +65,7 @@ impl FlatMapFunction for UnfoldOperator { } Ok(Box::new(res.into_iter())) } - EntryType::INTERSECT => { + EntryType::Intersection => { let intersection = entry .as_any_mut() .downcast_mut::() @@ -73,12 +73,12 @@ impl FlatMapFunction for UnfoldOperator { let mut res = Vec::with_capacity(intersection.len()); for item in intersection.drain() { let mut new_entry = input.clone(); - new_entry.append(IdEntry::new(item), self.alias); + new_entry.append(Vertex::new(item, None, DynDetails::default()), self.alias); res.push(new_entry); } Ok(Box::new(res.into_iter())) } - EntryType::PATH => Err(FnExecError::unsupported_error(&format!( + EntryType::Path => Err(FnExecError::unsupported_error(&format!( "unfold path entry {:?} in UnfoldOperator", entry )))?, diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs index acff7787d1ee..86825f31368d 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/auxilia.rs @@ -46,7 +46,7 @@ impl FilterMapFunction for AuxiliaOperator { // If queryable, then turn into graph element and do the query let graph = get_graph().ok_or(FnExecError::NullGraphError)?; let new_entry: Option = match entry.get_type() { - EntryType::VID | EntryType::VERTEX => { + EntryType::Vertex => { let id = entry.id(); let mut result_iter = graph.get_vertex(&[id], &self.query_params)?; result_iter.next().map(|mut vertex| { @@ -67,7 +67,7 @@ impl FilterMapFunction for AuxiliaOperator { DynEntry::new(vertex) }) } - EntryType::EDGE => { + EntryType::Edge => { let id = entry.id(); let mut result_iter = graph.get_edge(&[id], &self.query_params)?; result_iter.next().map(|mut edge| { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index 4d5ae169b2f7..c4d482c7c26d 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -152,7 +152,7 @@ impl FilterMapFunction for ExpandOrIntersect self.start_v_tag, input )))?; match entry.get_type() { - EntryType::VID | EntryType::VERTEX => { + EntryType::Vertex => { let id = entry.id(); let iter = self.stmt.exec(id)?.map(|e| { if let Some(vertex) = e.as_vertex() { diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs index 935a9381c599..443ae3d7a9ff 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/shuffle.rs @@ -51,11 +51,11 @@ impl RouteFunction for RecordRouter { fn route(&self, t: &Record) -> FnResult { if let Some(entry) = t.get(self.shuffle_key.clone()) { match entry.get_type() { - EntryType::VID | EntryType::VERTEX => { + EntryType::Vertex => { let id = entry.id(); Ok(self.p.get_partition(&id, self.num_workers)?) } - EntryType::EDGE => { + EntryType::Edge => { let e = entry .as_edge() .ok_or(FnExecError::Unreachable)?; @@ -63,7 +63,7 @@ impl RouteFunction for RecordRouter { .p .get_partition(&e.src_id, self.num_workers)?) } - EntryType::PATH => { + EntryType::Path => { let p = entry .as_graph_path() .ok_or(FnExecError::Unreachable)?; diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs index a0cae6fdbe71..82e28372c4f8 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/sink/sink.rs @@ -45,7 +45,7 @@ pub struct RecordSinkEncoder { impl RecordSinkEncoder { fn entry_to_pb(&self, e: &DynEntry) -> result_pb::Entry { let inner = match e.get_type() { - EntryType::COLLECTION => { + EntryType::Collection => { let collection = e .as_any_ref() .downcast_ref::() @@ -59,7 +59,7 @@ impl RecordSinkEncoder { collection: collection_pb, })) } - EntryType::INTERSECT => { + EntryType::Intersection => { let intersection = e .as_any_ref() .downcast_ref::() @@ -85,30 +85,26 @@ impl RecordSinkEncoder { fn element_to_pb(&self, e: &DynEntry) -> result_pb::Element { let inner = match e.get_type() { - EntryType::VID => { - let vertex_pb = self.vid_to_pb(&e.id()); - Some(result_pb::element::Inner::Vertex(vertex_pb)) - } - EntryType::VERTEX => { + EntryType::Vertex => { let vertex_pb = self.vertex_to_pb(e.as_vertex().unwrap()); Some(result_pb::element::Inner::Vertex(vertex_pb)) } - EntryType::EDGE => { + EntryType::Edge => { let edge_pb = self.edge_to_pb(e.as_edge().unwrap()); Some(result_pb::element::Inner::Edge(edge_pb)) } - EntryType::PATH => { + EntryType::Path => { let path_pb = self.path_to_pb(e.as_graph_path().unwrap()); Some(result_pb::element::Inner::GraphPath(path_pb)) } - EntryType::OBJECT => { + EntryType::Object => { let obj_pb = self.object_to_pb(e.as_object().unwrap().clone()); Some(result_pb::element::Inner::Object(obj_pb)) } - EntryType::COLLECTION => { + EntryType::Collection => { unreachable!() } - EntryType::INTERSECT => { + EntryType::Intersection => { unreachable!() } }; diff --git a/interactive_engine/executor/ir/runtime/src/process/record.rs b/interactive_engine/executor/ir/runtime/src/process/record.rs index 48106a79177f..4317de8107f8 100644 --- a/interactive_engine/executor/ir/runtime/src/process/record.rs +++ b/interactive_engine/executor/ir/runtime/src/process/record.rs @@ -137,8 +137,8 @@ impl Context for Record { .map(|entry| { let entry_type = entry.get_type(); match entry_type { - EntryType::COLLECTION => None, - EntryType::INTERSECT => None, + EntryType::Collection => None, + EntryType::Intersection => None, _ => Some(entry), } }) From 169e28f86369aa0391e4d2036d9034c95ff4c710 Mon Sep 17 00:00:00 2001 From: BingqingLyu Date: Wed, 18 Jan 2023 13:28:00 +0800 Subject: [PATCH 8/8] minor refine. --- .../ir/runtime/src/process/operator/map/expand_intersect.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs index c4d482c7c26d..8f2d1e479203 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/expand_intersect.rs @@ -42,6 +42,9 @@ struct ExpandOrIntersect { stmt: Box>, } +/// An optimized entry implementation for intersection, which denotes a collection of vertices; +/// Specifically, vertex_vec records the unique vertex ids in the collection, +/// and count_vec records the number of the corresponding vertex, since duplicated vertices are allowed. #[derive(Debug, Clone, Hash, PartialEq, PartialOrd)] pub struct IntersectionEntry { vertex_vec: Vec,